예제 #1
0
    def setup_attachments(self):
        '''
        Create ``_attachments``
        '''
        if '_attachments' not in self.doc:
            return

        attachdir = os.path.join(self.path, '_attachments')

        if not os.path.isdir(attachdir):
            self.setup_dir(attachdir)

        for filename in self.doc['_attachments']:
            filepath = os.path.normpath(self.locate_attach_dir(filename))
            currentdir = os.path.dirname(filepath)

            if not os.path.isdir(currentdir):
                self.setup_dir(currentdir)

            if self.signatures.get(filename) == util.sign(filepath):
                continue  # we already have the same file on fs

            self.dump_attachment(filename, filepath)

            logger.debug('clone attachment: "{0}"'.format(filename))
예제 #2
0
def clone(source, dest=None, rev=None):
    """
    Clone an application from a design_doc given.

    :attr design_doc: dict, the design doc retrieved from couchdb
    if something was wrong.
    """

    try:
        dburl, docid = source.split('_design/')
    except ValueError:
        raise AppError("%s isn't a valid source" % source)

    if not dest:
        dest = docid

    path = os.path.normpath(os.path.join(os.getcwd(), dest))
    if not os.path.exists(path):
        os.makedirs(path)

    db = client.Database(dburl[:-1], create=False)
    if not rev:
        doc = db.open_doc("_design/%s" % docid)
    else:
        doc = db.open_doc("_design/%s" % docid, rev=rev)
    docid = doc['_id']

    metadata = doc.get('couchapp', {})

    # get manifest
    manifest = metadata.get('manifest', {})

    # get signatures
    signatures = metadata.get('signatures', {})

    # get objects refs
    objects = metadata.get('objects', {})

    # create files from manifest
    if manifest:
        for filename in manifest:
            logger.debug("clone property: %s" % filename)
            filepath = os.path.join(path, filename)
            if filename.endswith('/'):
                if not os.path.isdir(filepath):
                    os.makedirs(filepath)
            elif filename == "couchapp.json":
                continue
            else:
                parts = util.split_path(filename)
                fname = parts.pop()
                v = doc
                while 1:
                    try:
                        for key in parts:
                            v = v[key]
                    except KeyError:
                        break
                    # remove extension
                    last_key, ext = os.path.splitext(fname)

                    # make sure key exist
                    try:
                        content = v[last_key]
                    except KeyError:
                        break

                    if isinstance(content, basestring):
                        _ref = md5(util.to_bytestring(content)).hexdigest()
                        if objects and _ref in objects:
                            content = objects[_ref]

                        if content.startswith('base64-encoded;'):
                            content = base64.b64decode(content[15:])

                    if fname.endswith('.json'):
                        content = util.json.dumps(content).encode('utf-8')

                    del v[last_key]

                    # make sure file dir have been created
                    filedir = os.path.dirname(filepath)
                    if not os.path.isdir(filedir):
                        os.makedirs(filedir)

                    util.write(filepath, content)

                    # remove the key from design doc
                    temp = doc
                    for key2 in parts:
                        if key2 == key:
                            if not temp[key2]:
                                del temp[key2]
                            break
                        temp = temp[key2]

    # second pass for missing key or in case
    # manifest isn't in app
    for key in doc.iterkeys():
        if key.startswith('_'):
            continue
        elif key in ('couchapp'):
            app_meta = copy.deepcopy(doc['couchapp'])
            if 'signatures' in app_meta:
                del app_meta['signatures']
            if 'manifest' in app_meta:
                del app_meta['manifest']
            if 'objects' in app_meta:
                del app_meta['objects']
            if 'length' in app_meta:
                del app_meta['length']
            if app_meta:
                couchapp_file = os.path.join(path, 'couchapp.json')
                util.write_json(couchapp_file, app_meta)
        elif key in ('views'):
            vs_dir = os.path.join(path, key)
            if not os.path.isdir(vs_dir):
                os.makedirs(vs_dir)
            for vsname, vs_item in doc[key].iteritems():
                vs_item_dir = os.path.join(vs_dir, vsname)
                if not os.path.isdir(vs_item_dir):
                    os.makedirs(vs_item_dir)
                for func_name, func in vs_item.iteritems():
                    filename = os.path.join(vs_item_dir, '%s.js' % func_name)
                    util.write(filename, func)
                    logger.warning("clone view not in manifest: %s" % filename)
        elif key in ('shows', 'lists', 'filter', 'updates'):
            showpath = os.path.join(path, key)
            if not os.path.isdir(showpath):
                os.makedirs(showpath)
            for func_name, func in doc[key].iteritems():
                filename = os.path.join(showpath, '%s.js' % func_name)
                util.write(filename, func)
                logger.warning("clone show or list not in manifest: %s" %
                               filename)
        else:
            filedir = os.path.join(path, key)
            if os.path.exists(filedir):
                continue
            else:
                logger.warning("clone property not in manifest: %s" % key)
                if isinstance(doc[key], (
                        list,
                        tuple,
                )):
                    util.write_json(filedir + ".json", doc[key])
                elif isinstance(doc[key], dict):
                    if not os.path.isdir(filedir):
                        os.makedirs(filedir)
                    for field, value in doc[key].iteritems():
                        fieldpath = os.path.join(filedir, field)
                        if isinstance(value, basestring):
                            if value.startswith('base64-encoded;'):
                                value = base64.b64decode(content[15:])
                            util.write(fieldpath, value)
                        else:
                            util.write_json(fieldpath + '.json', value)
                else:
                    value = doc[key]
                    if not isinstance(value, basestring):
                        value = str(value)
                    util.write(filedir, value)

    # save id
    idfile = os.path.join(path, '_id')
    util.write(idfile, doc['_id'])

    util.write_json(os.path.join(path, '.couchapprc'), {})

    if '_attachments' in doc:  # process attachments
        attachdir = os.path.join(path, '_attachments')
        if not os.path.isdir(attachdir):
            os.makedirs(attachdir)

        for filename in doc['_attachments'].iterkeys():
            if filename.startswith('vendor'):
                attach_parts = util.split_path(filename)
                vendor_attachdir = os.path.join(path, attach_parts.pop(0),
                                                attach_parts.pop(0),
                                                '_attachments')
                filepath = os.path.join(vendor_attachdir, *attach_parts)
            else:
                filepath = os.path.join(attachdir, filename)
            filepath = _replace_slash(filepath)
            currentdir = os.path.dirname(filepath)
            if not os.path.isdir(currentdir):
                os.makedirs(currentdir)

            if signatures.get(filename) != util.sign(filepath):
                resp = db.fetch_attachment(docid, filename)
                with open(filepath, 'wb') as f:
                    for chunk in resp.body_stream():
                        f.write(chunk)
                logger.debug("clone attachment: %s" % filename)

    logger.info("%s cloned in %s" % (source, dest))
예제 #3
0
    def doc(self, db=None, with_attachments=True, force=False):
        """ Function to reetrieve document object from
        document directory. If `with_attachments` is True
        attachments will be included and encoded"""

        manifest = []
        objects = {}
        signatures = {}
        attachments = {}

        self._doc = {"_id": self.docid}

        # get designdoc
        self._doc.update(self.dir_to_fields(self.docdir, manifest=manifest))

        if not "couchapp" in self._doc:
            self._doc["couchapp"] = {}

        self.olddoc = {}
        if db is not None:
            try:
                self.olddoc = db.open_doc(self._doc["_id"])
                attachments = self.olddoc.get("_attachments") or {}
                self._doc.update({"_rev": self.olddoc["_rev"]})
            except ResourceNotFound:
                self.olddoc = {}

        if "couchapp" in self.olddoc:
            old_signatures = self.olddoc["couchapp"].get("signatures", {})
        else:
            old_signatures = {}

        for name, filepath in self.attachments():
            signatures[name] = util.sign(filepath)
            if with_attachments and not old_signatures:
                logger.debug("attach %s ", name)
                attachments[name] = self.attachment_stub(name, filepath)

        if old_signatures:
            for name, signature in old_signatures.items():
                cursign = signatures.get(name)
                if not cursign:
                    logger.debug("detach %s ", name)
                    del attachments[name]
                elif cursign != signature:
                    logger.debug("detach %s ", name)
                    del attachments[name]
                else:
                    continue

            if with_attachments:
                for name, filepath in self.attachments():
                    if old_signatures.get(name) != signatures.get(name) or force:
                        logger.debug("attach %s ", name)
                        attachments[name] = self.attachment_stub(name, filepath)

        self._doc["_attachments"] = attachments

        self._doc["couchapp"].update({"manifest": manifest, "objects": objects, "signatures": signatures})

        if self.docid.startswith("_design/"):  # process macros
            for funs in ["shows", "lists", "updates", "filters", "spatial"]:
                if funs in self._doc:
                    package_shows(self._doc, self._doc[funs], self.docdir, objects)

            if "validate_doc_update" in self._doc:
                tmp_dict = {"validate_doc_update": self._doc["validate_doc_update"]}
                package_shows(self._doc, tmp_dict, self.docdir, objects)
                self._doc.update(tmp_dict)

            if "views" in self._doc:
                # clean views
                # we remove empty views and malformed from the list
                # of pushed views. We also clean manifest
                views = {}
                dmanifest = {}
                for i, fname in enumerate(manifest):
                    if fname.startswith("views/") and fname != "views/":
                        name, ext = os.path.splitext(fname)
                        if name.endswith("/"):
                            name = name[:-1]
                        dmanifest[name] = i

                for vname, value in self._doc["views"].iteritems():
                    if value and isinstance(value, dict):
                        views[vname] = value
                    else:
                        del manifest[dmanifest["views/%s" % vname]]
                self._doc["views"] = views
                package_views(self._doc, self._doc["views"], self.docdir, objects)

            if "fulltext" in self._doc:
                package_views(self._doc, self._doc["fulltext"], self.docdir, objects)

        return self._doc
예제 #4
0
파일: clone_app.py 프로젝트: Ivoz/couchapp
def clone(source, dest=None, rev=None):
    """
    Clone an application from a design_doc given.

    :attr design_doc: dict, the design doc retrieved from couchdb
    if something was wrong.
    """

    try:
        dburl, docid = source.split('_design/')
    except ValueError:
        raise AppError("%s isn't a valid source" % source)

    if not dest:
        dest = docid

    path = os.path.normpath(os.path.join(os.getcwd(), dest))
    if not os.path.exists(path):
        os.makedirs(path)

    db = client.Database(dburl[:-1], create=False)
    if not rev:
        doc = db.open_doc("_design/%s" % docid)
    else:
        doc = db.open_doc("_design/%s" % docid, rev=rev)
    docid = doc['_id']

    metadata = doc.get('couchapp', {})

    # get manifest
    manifest = metadata.get('manifest', {})

    # get signatures
    signatures = metadata.get('signatures', {})

    # get objects refs
    objects = metadata.get('objects', {})

    # create files from manifest
    if manifest:
        for filename in manifest:
            logger.debug("clone property: %s" % filename)
            filepath = os.path.join(path, filename)
            if filename.endswith('/'):
                if not os.path.isdir(filepath):
                    os.makedirs(filepath)
            elif filename == "couchapp.json":
                continue
            else:
                parts = util.split_path(filename)
                fname = parts.pop()
                v = doc
                while 1:
                    try:
                        for key in parts:
                            v = v[key]
                    except KeyError:
                        break
                    # remove extension
                    last_key, ext = os.path.splitext(fname)

                    # make sure key exist
                    try:
                        content = v[last_key]
                    except KeyError:
                        break

                    if isinstance(content, basestring):
                        _ref = md5(util.to_bytestring(content)).hexdigest()
                        if objects and _ref in objects:
                            content = objects[_ref]

                        if content.startswith('base64-encoded;'):
                            content = base64.b64decode(content[15:])

                    if fname.endswith('.json'):
                        content = util.json.dumps(content).encode('utf-8')

                    del v[last_key]

                    # make sure file dir have been created
                    filedir = os.path.dirname(filepath)
                    if not os.path.isdir(filedir):
                        os.makedirs(filedir)

                    util.write(filepath, content)

                    # remove the key from design doc
                    temp = doc
                    for key2 in parts:
                        if key2 == key:
                            if not temp[key2]:
                                del temp[key2]
                            break
                        temp = temp[key2]

    # second pass for missing key or in case
    # manifest isn't in app
    for key in doc.iterkeys():
        if key.startswith('_'):
            continue
        elif key in ('couchapp'):
            app_meta = copy.deepcopy(doc['couchapp'])
            if 'signatures' in app_meta:
                del app_meta['signatures']
            if 'manifest' in app_meta:
                del app_meta['manifest']
            if 'objects' in app_meta:
                del app_meta['objects']
            if 'length' in app_meta:
                del app_meta['length']
            if app_meta:
                couchapp_file = os.path.join(path, 'couchapp.json')
                util.write_json(couchapp_file, app_meta)
        elif key in ('views'):
            vs_dir = os.path.join(path, key)
            if not os.path.isdir(vs_dir):
                os.makedirs(vs_dir)
            for vsname, vs_item in doc[key].iteritems():
                vs_item_dir = os.path.join(vs_dir, vsname)
                if not os.path.isdir(vs_item_dir):
                    os.makedirs(vs_item_dir)
                for func_name, func in vs_item.iteritems():
                    filename = os.path.join(vs_item_dir, '%s.js' % func_name)
                    util.write(filename, func)
                    logger.warning("clone view not in manifest: %s" % filename)
        elif key in ('shows', 'lists', 'filter', 'update'):
            showpath = os.path.join(path, key)
            if not os.path.isdir(showpath):
                os.makedirs(showpath)
            for func_name, func in doc[key].iteritems():
                filename = os.path.join(showpath, '%s.js' % func_name)
                util.write(filename, func)
                logger.warning(
                    "clone show or list not in manifest: %s" % filename)
        else:
            filedir = os.path.join(path, key)
            if os.path.exists(filedir):
                continue
            else:
                logger.warning("clone property not in manifest: %s" % key)
                if isinstance(doc[key], (list, tuple,)):
                    util.write_json(filedir + ".json", doc[key])
                elif isinstance(doc[key], dict):
                    if not os.path.isdir(filedir):
                        os.makedirs(filedir)
                    for field, value in doc[key].iteritems():
                        fieldpath = os.path.join(filedir, field)
                        if isinstance(value, basestring):
                            if value.startswith('base64-encoded;'):
                                value = base64.b64decode(content[15:])
                            util.write(fieldpath, value)
                        else:
                            util.write_json(fieldpath + '.json', value)
                else:
                    value = doc[key]
                    if not isinstance(value, basestring):
                        value = str(value)
                    util.write(filedir, value)

    # save id
    idfile = os.path.join(path, '_id')
    util.write(idfile, doc['_id'])

    util.write_json(os.path.join(path, '.couchapprc'), {})

    if '_attachments' in doc:  # process attachments
        attachdir = os.path.join(path, '_attachments')
        if not os.path.isdir(attachdir):
            os.makedirs(attachdir)

        for filename in doc['_attachments'].iterkeys():
            if filename.startswith('vendor'):
                attach_parts = util.split_path(filename)
                vendor_attachdir = os.path.join(path, attach_parts.pop(0),
                                                attach_parts.pop(0),
                                                '_attachments')
                filepath = os.path.join(vendor_attachdir, *attach_parts)
            else:
                filepath = os.path.join(attachdir, filename)
            filepath = _replace_slash(filepath)
            currentdir = os.path.dirname(filepath)
            if not os.path.isdir(currentdir):
                os.makedirs(currentdir)

            if signatures.get(filename) != util.sign(filepath):
                resp = db.fetch_attachment(docid, filename)
                with open(filepath, 'wb') as f:
                    for chunk in resp.body_stream():
                        f.write(chunk)
                logger.debug("clone attachment: %s" % filename)

    logger.info("%s cloned in %s" % (source, dest))
예제 #5
0
    def doc(self, db=None, with_attachments=True):
        """ Function to reetrieve document object from
        document directory. If `with_attachments` is True
        attachments will be included and encoded"""
        
        manifest = []
        objects = {}
        self._doc = {'_id': self.docid}
        
        # get designdoc
        self._doc.update(self.dir_to_fields(self.docdir, manifest=manifest))
        
       
        if not 'couchapp' in self._doc:
             self._doc['couchapp'] = {}
            
        signatures = {}
        attachments = {}
        for name, filepath in self.attachments():
            signatures[name] = util.sign(filepath)
            if with_attachments:
                logger.debug("attach %s " % name)
                attachments[name] = {}
                with open(filepath, "rb") as f:
                    re_sp = re.compile('\s')
                    attachments[name]['data'] = re_sp.sub('', 
                                        base64.b64encode(f.read()))
                attachments[name]['content_type'] = ';'.join(filter(None, 
                                            mimetypes.guess_type(name)))
        
        if with_attachments: 
            self._doc['_attachments'] = attachments
            
        self._doc['couchapp'].update({
            'manifest': manifest,
            'objects': objects,
            'signatures': signatures
        })
        
        
        if self.docid.startswith('_design/'):  # process macros
            for funs in ['shows', 'lists', 'updates', 'filters', 
                    'fulltext']:
                if funs in self._doc:
                    package_shows(self._doc, self._doc[funs], self.docdir, 
                            objects)
            
            if 'validate_doc_update' in self._doc:
                tmp_dict = dict(validate_doc_update=self._doc[
                                                    "validate_doc_update"])
                package_shows( self._doc, tmp_dict, self.docdir, 
                    objects)
                self._doc.update(tmp_dict)

            if 'views' in  self._doc:
                # clean views
                # we remove empty views and malformed from the list
                # of pushed views. We also clean manifest
                views = {}
                dmanifest = {}
                for i, fname in enumerate(manifest):
                    if fname.startswith("views/") and fname != "views/":
                        name, ext = os.path.splitext(fname)
                        if name.endswith('/'):
                            name = name[:-1]
                        dmanifest[name] = i
            
                for vname, value in self._doc['views'].iteritems():
                    if value and isinstance(value, dict):
                        views[vname] = value
                    else:
                        del manifest[dmanifest["views/%s" % vname]]
                self._doc['views'] = views
                package_views(self._doc,self._doc["views"], self.docdir, 
                        objects)
        
        self.olddoc = {}
        if db is not None:
            try:
                self.olddoc = db.open_doc(self._doc['_id'])
                self._doc.update({'_rev': self.olddoc['_rev']})
            except ResourceNotFound:
                pass
        
        return self._doc
예제 #6
0
파일: localdoc.py 프로젝트: weijia/couchapp
    def doc(self, db=None, with_attachments=True, force=False):
        """
        Function to retrieve document object from document directory.

        :param with_attachments: If ``True``,
            attachments will be included and encoded
        """
        manifest = []
        objects = {}
        signatures = {}
        attachments = {}

        self._doc = {'_id': self.docid}

        # get designdoc
        self._doc.update(self.dir_to_fields(self.docdir, manifest=manifest))

        if not 'couchapp' in self._doc:
            self._doc['couchapp'] = {}

        self.olddoc = {}
        if db is not None:
            try:
                self.olddoc = db.open_doc(self._doc['_id'])
                attachments = self.olddoc.get('_attachments') or {}
                self._doc.update({'_rev': self.olddoc['_rev']})
            except ResourceNotFound:
                self.olddoc = {}

        if 'couchapp' in self.olddoc:
            old_signatures = self.olddoc['couchapp'].get('signatures', {})
        else:
            old_signatures = {}

        for name, filepath in self.attachments():
            signatures[name] = util.sign(filepath)
            if with_attachments and not old_signatures:
                logger.debug("attach %s ", name)
                attachments[name] = self.attachment_stub(name, filepath)

        if old_signatures:
            for name, signature in old_signatures.items():
                cursign = signatures.get(name)
                if not cursign:
                    logger.debug("detach %s ", name)
                    del attachments[name]
                elif cursign != signature:
                    logger.debug("detach %s ", name)
                    del attachments[name]
                else:
                    continue

            if with_attachments:
                for name, filepath in self.attachments():
                    if old_signatures.get(name) != \
                            signatures.get(name) or force:
                        logger.debug("attach %s ", name)
                        attachments[name] = self.attachment_stub(
                            name, filepath)

        self._doc['_attachments'] = attachments

        self._doc['couchapp'].update({
            'manifest': manifest,
            'objects': objects,
            'signatures': signatures
        })

        if self.docid.startswith('_design/'):  # process macros
            for funs in ['shows', 'lists', 'updates', 'filters', 'spatial']:
                if funs in self._doc:
                    package_shows(self._doc, self._doc[funs], self.docdir,
                                  objects)

            if 'validate_doc_update' in self._doc:
                tmp_dict = {
                    'validate_doc_update': self._doc["validate_doc_update"]
                }
                package_shows(self._doc, tmp_dict, self.docdir, objects)
                self._doc.update(tmp_dict)

            if 'views' in self._doc:
                # clean views
                # we remove empty views and malformed from the list
                # of pushed views. We also clean manifest
                views = {}
                dmanifest = {}
                for i, fname in enumerate(manifest):
                    if fname.startswith("views/") and fname != "views/":
                        name, ext = os.path.splitext(fname)
                        if name.endswith('/'):
                            name = name[:-1]
                        dmanifest[name] = i

                for vname, value in self._doc['views'].iteritems():
                    if value and isinstance(value, dict):
                        views[vname] = value
                    else:
                        del manifest[dmanifest["views/%s" % vname]]
                self._doc['views'] = views
                package_views(self._doc, self._doc["views"], self.docdir,
                              objects)

            if "fulltext" in self._doc:
                package_views(self._doc, self._doc["fulltext"], self.docdir,
                              objects)

        return self._doc