Exemple #1
0
 def _process_attachments(self, path, vendor=None):
     """ the function processing directory to yeld
     attachments. """
     if os.path.isdir(path):
         for root, dirs, files in os.walk(path):
             for dirname in dirs:
                 if dirname.startswith('.'):
                     dirs.remove(dirname)
                 elif self.check_ignore(dirname):
                     dirs.remove(dirname)
             if files:
                 for filename in files:
                     if filename.startswith('.'):
                         continue
                     elif self.check_ignore(filename):
                         continue
                     else:
                         filepath = os.path.join(root, filename)
                         name = utils.relpath(filepath, path)
                         if vendor is not None:
                             name = os.path.join('vendor', vendor, name)
                         name = _replace_backslash(name)
                         yield (name, filepath)
Exemple #2
0
    def dir_to_fields(self, current_dir='', depth=0,
                manifest=[]):
        """ process a directory and get all members """        
        
        fields={}
        if not current_dir:
            current_dir = self.docdir
        for name in os.listdir(current_dir):
            current_path = os.path.join(current_dir, name)
            rel_path = _replace_backslash(utils.relpath(current_path, self.docdir))
            if name.startswith("."):
                continue
            elif self.check_ignore(name):
                continue
            elif depth == 0 and name.startswith('_'):
                # files starting with "_" are always "special"
                continue
            elif name == '_attachments':
                continue
            elif depth == 0 and (name == 'couchapp' or name == 'couchapp.json'):
                # we are in app_meta
                if name == "couchapp":
                    manifest.append('%s/' % rel_path)
                    content = self.dir_to_fields(current_path,
                        depth=depth+1, manifest=manifest)
                else:
                    manifest.append(rel_path)
                    content = utils.read_json(current_path)
                    if not isinstance(content, dict):
                        content = { "meta": content }
                if 'signatures' in content:
                    del content['signatures']

                if 'manifest' in content:
                    del content['manifest']

                if 'objects' in content:
                    del content['objects']
                
                if 'length' in content:
                    del content['length']

                if 'couchapp' in fields:
                    fields['couchapp'].update(content)
                else:
                    fields['couchapp'] = content
            elif os.path.isdir(current_path):
                manifest.append('%s/' % rel_path)
                fields[name] = self.dir_to_fields(current_path,
                        depth=depth+1, manifest=manifest)
            else:
                logger.debug("push %s" % rel_path)
                  
                content = ''  
                if name.endswith('.json'):
                    try:
                        content = utils.read_json(current_path)
                    except ValueError:
                        logger.error("Json invalid in %s" % current_path)           
                else:
                    try:
                        content = utils.read_file(current_path).strip()
                    except UnicodeDecodeError:
                        logger.warning("%s isn't encoded in utf8" % current_path)
                        content = utils.read_file(current_path, utf8=False)
                        try:
                            content.encode('utf-8')
                        except UnicodeError:
                            logger.warning(
                            "plan B didn't work, %s is a binary" % current_path)
                            logger.warning("use plan C: encode to base64")   
                            content = "base64-encoded;%s" % base64.b64encode(
                                                                        content)

                
                # remove extension
                name, ext = os.path.splitext(name)
                if name in fields:
                    logger.warning(
        "%(name)s is already in properties. Can't add (%(fqn)s)" % {
                            "name": name, "fqn": rel_path })
                else:
                    manifest.append(rel_path)
                    fields[name] = content
        return fields