Ejemplo n.º 1
0
    def _encode_content(name, path):
        '''
        This is a private subroutine for ``dir_to_fields``
        '''
        if name.endswith('.json'):
            try:
                return util.read_json(path, raise_on_error=True)
            except ValueError:
                logger.error("Json invalid in %s", path)
                return ''

        try:
            content = util.read(path)
        except UnicodeDecodeError:
            logger.warning("%s isn't encoded in utf8", path)
            content = util.read(path, utf8=False)

            try:
                content.encode('utf-8')
            except UnicodeError:
                logger.warning("plan B didn't work, %s is a binary", path)
                logger.warning("use plan C: encode to base64")
                content = ("base64-encoded;%s" % base64.b64encode(content))

        return content
Ejemplo n.º 2
0
    def _encode_content(name, path):
        '''
        This is a private subroutine for ``dir_to_fields``
        '''
        if name.endswith('.json'):
            try:
                return util.read_json(path, raise_on_error=True)
            except ValueError:
                logger.error("Json invalid in %s", path)
                return ''

        try:
            content = util.read(path)
        except UnicodeDecodeError:
            logger.warning("%s isn't encoded in utf8", path)
            content = util.read(path, utf8=False)

            try:
                content.encode('utf-8')
            except UnicodeError:
                logger.warning("plan B didn't work, %s is a binary", path)
                logger.warning("use plan C: encode to base64")
                content = ("base64-encoded;%s" % base64.b64encode(content))

        return content
Ejemplo n.º 3
0
 def rjson(mo):
     if mo.group(2).startswith('_attachments'):
         # someone  want to include from attachments
         path = os.path.join(app_dir, mo.group(2).strip())
         filenum = 0
         for filename in glob.iglob(path):
             logger.debug("process json macro: %s" % filename)
             library = ''
             try:
                 if filename.endswith('.json'):
                     library = util.read_json(filename)
                 else:
                     library = util.read(filename)
             except IOError, e:
                 raise MacroError(str(e))
             filenum += 1
             current_file = filename.split(app_dir)[1]
             fields = current_file.split('/')
             count = len(fields)
             include_to = included
             for i, field in enumerate(fields):
                 if i + 1 < count:
                     include_to[field] = {}
                     include_to = include_to[field]
                 else:
                     include_to[field] = library
         if not filenum:
             raise MacroError("Processing code: No file matching '%s'" %
                              mo.group(2))
Ejemplo n.º 4
0
 def rjson(mo):
     if mo.group(2).startswith('_attachments'):
         # someone  want to include from attachments
         path = os.path.join(app_dir, mo.group(2).strip())
         filenum = 0
         for filename in glob.iglob(path):
             logger.debug("process json macro: %s" % filename)
             library = ''
             try:
                 if filename.endswith('.json'):
                     library = util.read_json(filename)
                 else:
                     library = util.read(filename)
             except IOError, e:
                 raise MacroError(str(e))
             filenum += 1
             current_file = filename.split(app_dir)[1]
             fields = current_file.split('/')
             count = len(fields)
             include_to = included
             for i, field in enumerate(fields):
                 if i+1 < count:
                     include_to[field] = {}
                     include_to = include_to[field]
                 else:
                     include_to[field] = library
         if not filenum:
             raise MacroError("Processing code: No file matching '%s'" %
                              mo.group(2))
    def compress_css(self, css):
        re_url = re.compile('url\s*\(([^\s"].*)\)')

        src_fpath = ''
        fname_dir = ''

        def replace_url(mo):
            """ make sure urls are relative to css path """
            css_url = mo.group(0)[4:].strip(")").replace("'",
                                                         "").replace('"', '')
            css_path = os.path.join(os.path.dirname(src_fpath), css_url)

            rel_path = util.relpath(css_path, fname_dir)
            return "url(%s)" % rel_path

        for fname, src_files in css.iteritems():
            output_css = ''

            dest_path = os.path.join(self.attach_dir, fname)
            fname_dir = os.path.dirname(dest_path)

            for src_fname in src_files:
                src_fpath = os.path.join(self.appdir, src_fname)

                if os.path.exists(src_fpath):
                    content_css = str(
                        compress_css.CSSParser(util.read(src_fpath)))
                    content_css = re_url.sub(replace_url, content_css)
                    output_css += content_css
                    logger.debug("Merging %s in %s" % (src_fname, fname))

            if not os.path.isdir(fname_dir):
                os.makedirs(fname_dir)
            util.write(dest_path, output_css)
Ejemplo n.º 6
0
    def compress_css(self, css):
        re_url = re.compile('url\s*\(([^\s"].*)\)')

        src_fpath = ''
        fname_dir = ''

        def replace_url(mo):
            """ make sure urls are relative to css path """
            css_url = mo.group(0)[4:].strip(")").replace("'", "").replace('"',
                                                                          '')
            css_path = os.path.join(os.path.dirname(src_fpath), css_url)

            rel_path = util.relpath(css_path, fname_dir)
            return "url(%s)" % rel_path

        for fname, src_files in css.iteritems():
            output_css = ''

            dest_path = os.path.join(self.attach_dir, fname)
            fname_dir = os.path.dirname(dest_path)

            for src_fname in src_files:
                src_fpath = os.path.join(self.appdir, src_fname)

                if os.path.exists(src_fpath):
                    content_css = \
                        str(compress_css.CSSParser(util.read(src_fpath)))
                    content_css = re_url.sub(replace_url, content_css)
                    output_css += content_css
                    logger.debug("Merging %s in %s" % (src_fname, fname))

            if not os.path.isdir(fname_dir):
                os.makedirs(fname_dir)
            util.write(dest_path, output_css)
Ejemplo n.º 7
0
 def get_id(self):
     """
     if there is an _id file, docid is extracted from it,
     else we take the current folder name.
     """
     idfile = os.path.join(self.docdir, '_id')
     if os.path.exists(idfile):
         docid = util.read(idfile).split("\n")[0].strip()
         if docid: return docid
     return "_design/%s" % os.path.split(self.docdir)[1]
Ejemplo n.º 8
0
    def get_id(self):
        """
        if there is an ``_id`` file, docid is extracted from it,
        else we take the current folder name.
        """
        idfile = os.path.join(self.docdir, '_id')
        if os.path.exists(idfile):
            docid = util.read(idfile).split("\n")[0].strip()
            if docid:
                return docid

        dirname = os.path.split(self.docdir)[1]
        return '_design/%s' % dirname if self.is_ddoc else dirname
Ejemplo n.º 9
0
 def rreq(mo):
     # just read the file and return it
     path = os.path.join(app_dir, mo.group(2).strip())
     library = ''
     filenum = 0
     for filename in glob.iglob(path):
         logger.debug("process code macro: %s" % filename)
         try:
             cnt = util.read(filename)
             if cnt.find("!code") >= 0:
                 cnt = run_code_macros(cnt, app_dir)
             library += cnt
         except IOError, e:
             raise MacroError(str(e))
         filenum += 1
Ejemplo n.º 10
0
 def rreq(mo):
     # just read the file and return it
     path = os.path.join(app_dir, mo.group(2).strip())
     library = ''
     filenum = 0
     for filename in glob.iglob(path):
         logger.debug("process code macro: %s" % filename)
         try:
             cnt = util.read(filename)
             if cnt.find("!code") >= 0:
                 cnt = run_code_macros(cnt, app_dir)
             library += cnt
         except IOError, e:
             raise MacroError(str(e))
         filenum += 1
    def compress_js(self, backend, js):
        logger.info("compress js with %s " % backend.__about__)

        for fname, src_files in js.iteritems():
            output_js = ''

            dest_path = os.path.join(self.attach_dir, fname)
            fname_dir = os.path.dirname(dest_path)

            for src_fname in src_files:
                src_fpath = os.path.join(self.appdir, src_fname)
                if os.path.isfile(src_fpath):
                    output_js += "/* %s */\n" % src_fpath
                    output_js += util.read(src_fpath)
                    logger.debug("merging %s in %s" % (src_fname, fname))

            if not os.path.isdir(fname_dir):
                os.makedirs(fname_dir)

            output_js = backend.compress(output_js)
            util.write(dest_path, output_js)
Ejemplo n.º 12
0
    def compress_js(self, backend, js):
        logger.info("compress js with %s " % backend.__about__)

        for fname, src_files in js.iteritems():
            output_js = ''

            dest_path = os.path.join(self.attach_dir, fname)
            fname_dir = os.path.dirname(dest_path)

            for src_fname in src_files:
                src_fpath = os.path.join(self.appdir, src_fname)
                if os.path.isfile(src_fpath):
                    output_js += "/* %s */\n" % src_fpath
                    output_js += util.read(src_fpath)
                    logger.debug("merging %s in %s" % (src_fname, fname))

            if not os.path.isdir(fname_dir):
                os.makedirs(fname_dir)

            output_js = backend.compress(output_js)
            util.write(dest_path, output_js)
Ejemplo n.º 13
0
    def dir_to_fields(self, current_dir="", depth=0, manifest=[]):
        """ process a directory and get all members """

        fields = {}
        if not current_dir:
            current_dir = self.docdir
        for name in os.listdir(current_dir):
            current_path = os.path.join(current_dir, name)
            rel_path = _replace_backslash(util.relpath(current_path, self.docdir))
            if name.startswith("."):
                continue
            elif self.check_ignore(name):
                continue
            elif depth == 0 and name.startswith("_"):
                # files starting with "_" are always "special"
                continue
            elif name == "_attachments":
                continue
            elif depth == 0 and (name == "couchapp" or name == "couchapp.json"):
                # we are in app_meta
                if name == "couchapp":
                    manifest.append("%s/" % rel_path)
                    content = self.dir_to_fields(current_path, depth=depth + 1, manifest=manifest)
                else:
                    manifest.append(rel_path)
                    content = util.read_json(current_path)
                    if not isinstance(content, dict):
                        content = {"meta": content}
                if "signatures" in content:
                    del content["signatures"]

                if "manifest" in content:
                    del content["manifest"]

                if "objects" in content:
                    del content["objects"]

                if "length" in content:
                    del content["length"]

                if "couchapp" in fields:
                    fields["couchapp"].update(content)
                else:
                    fields["couchapp"] = content
            elif os.path.isdir(current_path):
                manifest.append("%s/" % rel_path)
                fields[name] = self.dir_to_fields(current_path, depth=depth + 1, manifest=manifest)
            else:
                logger.debug("push %s", rel_path)

                content = ""
                if name.endswith(".json"):
                    try:
                        content = util.read_json(current_path)
                    except ValueError:
                        logger.error("Json invalid in %s", current_path)
                else:
                    try:
                        content = util.read(current_path).strip()
                    except UnicodeDecodeError:
                        logger.warning("%s isn't encoded in utf8", current_path)
                        content = util.read(current_path, utf8=False)
                        try:
                            content.encode("utf-8")
                        except UnicodeError:
                            logger.warning("plan B didn't work, " "%s is a binary", current_path)
                            logger.warning("use plan C: encode to base64")
                            content = "base64-encoded;%s" % base64.b64encode(content)

                # remove extension
                name, ext = os.path.splitext(name)
                if name in fields:
                    logger.warning(
                        "%(name)s is already in properties. " "Can't add (%(fqn)s)", {"name": name, "fqn": rel_path}
                    )
                else:
                    manifest.append(rel_path)
                    fields[name] = content
        return fields
Ejemplo n.º 14
0
    def dir_to_fields(self, current_dir='', depth=0,
                manifest=[]):
        """ process a directory and get all members """        
        
        fields={}
        if not current_dir:
            current_dir = self.docdir
        for name in os.listdir(current_dir):
            current_path = os.path.join(current_dir, name)
            rel_path = _replace_backslash(util.relpath(current_path, self.docdir))
            if name.startswith("."):
                continue
            elif self.check_ignore(name):
                continue
            elif depth == 0 and name.startswith('_'):
                # files starting with "_" are always "special"
                continue
            elif name == '_attachments':
                continue
            elif depth == 0 and (name == 'couchapp' or name == 'couchapp.json'):
                # we are in app_meta
                if name == "couchapp":
                    manifest.append('%s/' % rel_path)
                    content = self.dir_to_fields(current_path,
                        depth=depth+1, manifest=manifest)
                else:
                    manifest.append(rel_path)
                    content = util.read_json(current_path)
                    if not isinstance(content, dict):
                        content = { "meta": content }
                if 'signatures' in content:
                    del content['signatures']

                if 'manifest' in content:
                    del content['manifest']

                if 'objects' in content:
                    del content['objects']
                
                if 'length' in content:
                    del content['length']

                if 'couchapp' in fields:
                    fields['couchapp'].update(content)
                else:
                    fields['couchapp'] = content
            elif os.path.isdir(current_path):
                manifest.append('%s/' % rel_path)
                fields[name] = self.dir_to_fields(current_path,
                        depth=depth+1, manifest=manifest)
            else:
                logger.debug("push %s" % rel_path)
                  
                content = ''  
                if name.endswith('.json'):
                    try:
                        content = util.read_json(current_path)
                    except ValueError:
                        logger.error("Json invalid in %s" % current_path)           
                else:
                    try:
                        content = util.read(current_path).strip()
                    except UnicodeDecodeError, e:
                        logger.warning("%s isn't encoded in utf8" % current_path)
                        content = util.read(current_path, utf8=False)
                        try:
                            content.encode('utf-8')
                        except UnicodeError, e:
                            logger.warning(
                            "plan B didn't work, %s is a binary" % current_path)
                            logger.warning("use plan C: encode to base64")   
                            content = "base64-encoded;%s" % base64.b64encode(
                                                                        content)

                
                # remove extension
                name, ext = os.path.splitext(name)
                if name in fields and ext in ('.txt'):
                    logger.warning(
        "%(name)s is already in properties. Can't add (%(name)s%(ext)s)" % {
                            "name": name, "ext": ext })
                else:
                    manifest.append(rel_path)
                    fields[name] = content
Ejemplo n.º 15
0
    def dir_to_fields(self, current_dir='', depth=0,
                manifest=[]):
        """ process a directory and get all members """        
        
        fields={}
        if not current_dir:
            current_dir = self.docdir
        for name in os.listdir(current_dir):
            current_path = os.path.join(current_dir, name)
            rel_path = _replace_backslash(util.relpath(current_path, self.docdir))
            if name.startswith("."):
                continue
            elif self.check_ignore(name):
                continue
            elif depth == 0 and name.startswith('_'):
                # files starting with "_" are always "special"
                continue
            elif name == '_attachments':
                continue
            elif depth == 0 and (name == 'couchapp' or name == 'couchapp.json'):
                # we are in app_meta
                if name == "couchapp":
                    manifest.append('%s/' % rel_path)
                    content = self.dir_to_fields(current_path,
                        depth=depth+1, manifest=manifest)
                else:
                    manifest.append(rel_path)
                    content = util.read_json(current_path)
                    if not isinstance(content, dict):
                        content = { "meta": content }
                if 'signatures' in content:
                    del content['signatures']

                if 'manifest' in content:
                    del content['manifest']

                if 'objects' in content:
                    del content['objects']
                
                if 'length' in content:
                    del content['length']

                if 'couchapp' in fields:
                    fields['couchapp'].update(content)
                else:
                    fields['couchapp'] = content
            elif os.path.isdir(current_path):
                manifest.append('%s/' % rel_path)
                fields[name] = self.dir_to_fields(current_path,
                        depth=depth+1, manifest=manifest)
            else:
                logger.debug("push %s" % rel_path)
                  
                content = ''  
                if name.endswith('.json'):
                    try:
                        content = util.read_json(current_path)
                    except ValueError:
                        logger.error("Json invalid in %s" % current_path)           
                else:
                    try:
                        content = util.read(current_path).strip()
                    except UnicodeDecodeError, e:
                        logger.warning("%s isn't encoded in utf8" % current_path)
                        content = util.read(current_path, utf8=False)
                        try:
                            content.encode('utf-8')
                        except UnicodeError, e:
                            logger.warning(
                            "plan B didn't work, %s is a binary" % current_path)
                            logger.warning("use plan C: encode to base64")   
                            content = "base64-encoded;%s" % base64.b64encode(
                                                                        content)

                
                # remove extension
                name, ext = os.path.splitext(name)
                if name in fields:
                    logger.warning(
        "%(name)s is already in properties. Can't add (%(fqn)s)" % {
                            "name": name, "fqn": rel_path })
                else:
                    manifest.append(rel_path)
                    fields[name] = content