def _encode_options(self, options): retval = {} for name, value in options.items(): if name in ('key', 'startkey', 'endkey') \ or not isinstance(value, basestring): value = json.dumps(value, allow_nan=False, ensure_ascii=False) retval[name] = value return retval
def write_json(self, fname, content): """ serialize content in json and save it :attr fname: string :attr content: string """ self.write(fname, json.dumps(content))
def _exec(self, options): body = {'map': self.map_fun, 'language': self.language} if self.reduce_fun: body['reduce'] = self.reduce_fun if 'keys' in options: options = options.copy() body['keys'] = options.pop('keys') content = json.dumps(body, allow_nan=False, ensure_ascii=False).encode('utf-8') resp, data = self.resource.post(content=content, headers={ 'Content-Type': 'application/json' }, **self._encode_options(options)) return data
def _request(self, method, path=None, content=None, headers=None, **params): from couchapp import __version__ headers = headers or {} headers.setdefault('Accept', 'application/json') headers.setdefault('User-Agent', 'couchapp %s' % __version__) body = None if content is not None: if not isinstance(content, basestring): body = json.dumps(content, allow_nan=False, ensure_ascii=False).encode('utf-8') headers.setdefault('Content-Type', 'application/json') else: body = content headers.setdefault('Content-Length', str(len(body))) def _make_request(retry=1): try: return self.http.request(uri(self.uri, path, **params), method, body=body, headers=headers) except socket.error, e: if retry > 0 and e.args[0] == 54: # reset by peer return _make_request(retry - 1) raise
if not filenum: raise MacroError("Processing code: No file matching '%s'" % mo.group(2)) else: fields = mo.group(2).split('.') library = doc count = len(fields) include_to = included for i, field in enumerate(fields): if not field in library: break library = library[field] if i+1 < count: include_to[field] = include_to.get(field, {}) include_to = include_to[field] else: include_to[field] = library return f_string def rjson2(mo): return '\n'.join(varstrings) re_json = re.compile('(\/\/|#)\ ?!json (.*)') re_json.sub(rjson, f_string) if not included: return f_string for k, v in included.iteritems(): varstrings.append("var %s = %s;" % (k, json.dumps(v))) return re_json.sub(rjson2, f_string)
def designdoc_to_fs(self, db, design_doc): """ Clone an application from a design_doc given. :attr design_doc: dict, the design doc retrieved from couchdb if something was wrong. """ app_name = get_appname(design_doc["_id"]) docid = design_doc["_id"] metadata = design_doc.get("couchapp", {}) # get manifest manifest = metadata.get("manifest", {}) # get signatures signatures = metadata.get("signatures", {}) # get objects refs objects = metadata.get("objects", {}) # create files from manifest if manifest: for filename in manifest: if self.ui.verbose >= 2: print "clone property: %s" % filename file_path = self.ui.rjoin(self.app_dir, filename) if filename.endswith("/"): if not self.ui.isdir(file_path): self.ui.makedirs(file_path) elif filename == "couchapp.json": continue else: parts = filename.split("/") fname = parts.pop() v = design_doc while 1: try: for key in parts: v = v[key] except KeyError: break # remove extension last_key, ext = os.path.splitext(fname) # make sure key exist try: content = v[last_key] except KeyError: break if isinstance(content, basestring): _ref = md5(to_bytestring(content)).hexdigest() if objects and _ref in objects: content = objects[_ref] if fname.endswith(".json"): content = json.dumps(content) del v[last_key] # make sure file dir have been created file_dir = self.ui.dirname(file_path) if not self.ui.isdir(file_dir): self.ui.makedirs(file_dir) self.ui.write(file_path, content) # remove the key from design doc temp = design_doc for key2 in parts: if key2 == key: if not temp[key2]: del temp[key2] break temp = temp[key2] # second pass for missing key or in case # manifest isn't in app for key in design_doc.iterkeys(): if key.startswith("_"): continue elif key in ("couchapp"): app_meta = copy.deepcopy(design_doc["couchapp"]) if "signatures" in app_meta: del app_meta["signatures"] if "manifest" in app_meta: del app_meta["manifest"] if "objects" in app_meta: del app_meta["objects"] if "lenght" in app_meta: del app_meta["lenght"] if app_meta: couchapp_file = self.ui.rjoin(self.app_dir, "couchapp.json") self.ui.write_json(couchapp_file, app_meta) elif key in ("views"): vs_dir = self.ui.rjoin(self.app_dir, key) if not self.ui.isdir(vs_dir): self.ui.makedirs(vs_dir) for vsname, vs_item in design_doc[key].iteritems(): vs_item_dir = self.ui.rjoin(vs_dir, vsname) if not self.ui.isdir(vs_item_dir): self.ui.makedirs(vs_item_dir) for func_name, func in vs_item.iteritems(): filename = self.ui.rjoin(vs_item_dir, "%s.js" % func_name) self.ui.write(filename, func) if self.ui.verbose >= 2: self.ui.logger.info("clone view not in manifest: %s" % filename) elif key in ("shows", "lists"): dir = self.ui.rjoin(self.app_dir, key) if not self.ui.isdir(dir): self.ui.makedirs(dir) for func_name, func in design_doc[key].iteritems(): filename = self.ui.rjoin(dir, "%s.js" % func_name) self.ui.write(filename, func) if self.ui.verbose >= 2: self.ui.logger.info("clone show or list not in manifest: %s" % filename) else: file_dir = self.ui.rjoin(self.app_dir, key) if self.ui.exists(file_dir): continue else: if self.ui.verbose >= 2: self.ui.logger.info("clone property not in manifest: %s" % key) if isinstance(design_doc[key], (list, tuple)): self.ui.write_json(file_dir + ".json", design[key]) elif isinstance(design_doc[key], dict): if not self.ui.isdir(file_dir): self.ui.makedirs(file_dir) for field, value in design_doc[key].iteritems(): field_path = self.ui.rjoin(file_dir, field) if isinstance(value, basestring): self.ui.write(field_path, value) else: self.ui.write_json(field_path + ".json", value) else: value = design_doc[key] if not isinstance(value, basestring): value = str(value) self.ui.write(file_dir, value) # get attachments if "_attachments" in design_doc: attach_dir = self.ui.rjoin(self.app_dir, "_attachments") if not self.ui.isdir(attach_dir): self.ui.makedirs(attach_dir) for filename in design_doc["_attachments"].iterkeys(): if filename.startswith("vendor"): attach_parts = filename.split("/") vendor_attach_dir = self.ui.rjoin( self.app_dir, attach_parts.pop(0), attach_parts.pop(0), "_attachments" ) file_path = self.ui.rjoin(vendor_attach_dir, "/".join(attach_parts)) else: file_path = self.ui.rjoin(attach_dir, filename) current_dir = self.ui.dirname(file_path) if not self.ui.isdir(current_dir): self.ui.makedirs(current_dir) if signatures.get(filename) != self.ui.sign(file_path): self.ui.write(file_path, db.get_attachment(docid, filename)) if self.ui.verbose >= 2: self.ui.logger.info("clone attachment: %s" % filename)
def designdoc_to_fs(self, db, design_doc): """ Clone an application from a design_doc given. :attr design_doc: dict, the design doc retrieved from couchdb if something was wrong. """ app_name = get_appname(design_doc['_id']) docid = design_doc['_id'] metadata = design_doc.get('couchapp', {}) # get manifest manifest = metadata.get('manifest', {}) # get signatures signatures = metadata.get('signatures', {}) # get objects refs objects = metadata.get('objects', {}) # create files from manifest if manifest: for filename in manifest: if self.ui.verbose >=2: self.ui.logger.info("clone property: %s" % filename) file_path = self.ui.rjoin(self.app_dir, filename) if filename.endswith('/'): if not self.ui.isdir(file_path): self.ui.makedirs(file_path) elif filename == "couchapp.json": continue else: parts = self.ui.split_path(filename) fname = parts.pop() v = design_doc while 1: try: for key in parts: v = v[key] except KeyError: break # remove extension last_key, ext = os.path.splitext(fname) # make sure key exist try: content = v[last_key] except KeyError: break if isinstance(content, basestring): _ref = md5(to_bytestring(content)).hexdigest() if objects and _ref in objects: content = objects[_ref] if content.startswith('base64-encoded;'): content = base64.b64decode(content[15:]) if fname.endswith('.json'): content = json.dumps(content) del v[last_key] # make sure file dir have been created file_dir = self.ui.dirname(file_path) if not self.ui.isdir(file_dir): self.ui.makedirs(file_dir) self.ui.write(file_path, content) # remove the key from design doc temp = design_doc for key2 in parts: if key2 == key: if not temp[key2]: del temp[key2] break temp = temp[key2] # second pass for missing key or in case # manifest isn't in app for key in design_doc.iterkeys(): if key.startswith('_'): continue elif key in ('couchapp'): app_meta = copy.deepcopy(design_doc['couchapp']) if 'signatures' in app_meta: del app_meta['signatures'] if 'manifest' in app_meta: del app_meta['manifest'] if 'objects' in app_meta: del app_meta['objects'] if 'lenght' in app_meta: del app_meta['lenght'] if app_meta: couchapp_file = self.ui.rjoin(self.app_dir, 'couchapp.json') self.ui.write_json(couchapp_file, app_meta) elif key in ('views'): vs_dir = self.ui.rjoin(self.app_dir, key) if not self.ui.isdir(vs_dir): self.ui.makedirs(vs_dir) for vsname, vs_item in design_doc[key].iteritems(): vs_item_dir = self.ui.rjoin(vs_dir, vsname) if not self.ui.isdir(vs_item_dir): self.ui.makedirs(vs_item_dir) for func_name, func in vs_item.iteritems(): filename = self.ui.rjoin(vs_item_dir, '%s.js' % func_name) self.ui.write(filename, func) if self.ui.verbose >=2: self.ui.logger.info("clone view not in manifest: %s" % filename) elif key in ('shows', 'lists'): show_path = self.ui.rjoin(self.app_dir, key) if not self.ui.isdir(show_path): self.ui.makedirs(show_path) for func_name, func in design_doc[key].iteritems(): filename = self.ui.rjoin(show_path, '%s.js' % func_name) self.ui.write(filename, func) if self.ui.verbose >=2: self.ui.logger.info("clone show or list not in manifest: %s" % filename) else: file_dir = self.ui.rjoin(self.app_dir, key) if self.ui.exists(file_dir): continue else: if self.ui.verbose >=2: self.ui.logger.info("clone property not in manifest: %s" % key) if isinstance(design_doc[key], (list, tuple,)): self.ui.write_json(file_dir + ".json", design[key]) elif isinstance(design_doc[key], dict): if not self.ui.isdir(file_dir): self.ui.makedirs(file_dir) for field, value in design_doc[key].iteritems(): field_path = self.ui.rjoin(file_dir, field) if isinstance(value, basestring): if value.startswith('base64-encoded;'): value = base64.b64decode(content[15:]) self.ui.write(field_path, value) else: self.ui.write_json(field_path + '.json', value) else: value = design_doc[key] if not isinstance(value, basestring): value = str(value) self.ui.write(file_dir, value) # get attachments if '_attachments' in design_doc: attach_dir = self.ui.rjoin(self.app_dir, '_attachments') if not self.ui.isdir(attach_dir): self.ui.makedirs(attach_dir) for filename in design_doc['_attachments'].iterkeys(): if filename.startswith('vendor'): attach_parts = self.ui.split_path(filename) vendor_attach_dir = self.ui.rjoin(self.app_dir, attach_parts.pop(0), attach_parts.pop(0), '_attachments') file_path = self.ui.rjoin(vendor_attach_dir, *attach_parts) else: file_path = self.ui.rjoin(attach_dir, filename) current_dir = self.ui.dirname(file_path) if not self.ui.isdir(current_dir): self.ui.makedirs(current_dir) if signatures.get(filename) != self.ui.sign(file_path): self.ui.write(file_path, db.get_attachment(docid, filename)) if self.ui.verbose>=2: self.ui.logger.info("clone attachment: %s" % filename)
def push(self, dbstring, app_name, **kwargs): """Pushes the app specified to the CouchDB instance :attr dbstring: string, db url or db environment name. :attr app_name: name of app to push. :attr verbose: boolean, default is False """ self.extensions.notify("pre-push", self.ui, self) app_name = self.ui.get_app_name(dbstring, app_name) design_doc = self.fs_to_designdoc(app_name) docid = design_doc['_id'] attach_dir = self.ui.rjoin(self.app_dir, '_attachments') new_doc = copy.deepcopy(design_doc) couchapp = design_doc.get('couchapp', {}) if couchapp: index = couchapp.get('index', False) else: index = False new_doc['couchapp'] = {} # get docs from _docs folder docs_dir = self.ui.rjoin(self.app_dir, '_docs') if self.ui.isdir(docs_dir): docs = self.fs_to_docs(docs_dir) else: docs = [] # do we export ? if kwargs.get('export', False): # process attachments design = copy.deepcopy(design_doc) del new_doc['_attachments'] if not 'couchapp' in design: design['couchapp'] = {} attachments = design['_attachments'] _length = design['couchapp'].get('length', {}) new_attachments = {} for filename, value in attachments.iteritems(): content_length = _length.get(filename, None) if self.ui.verbose >= 2: self.ui.logger.info("Attaching %s (%s)" % (filename, content_length)) f = open(value, "rb") # fix issue with httplib that raises BadStatusLine # error because it didn't close the connection new_attachments[filename] = { "content_type": ';'.join(filter(None, mimetypes.guess_type(filename))), "data": base64.b64encode(f.read()), } # update signatures if not 'couchapp' in new_doc: new_doc['couchapp'] = {} new_doc['_attachments'] = new_attachments if kwargs.get('output', None) is not None: self.ui.write_json(kwargs.get('output'), new_doc) else: print json.dumps(new_doc) self.extensions.notify("post-push", self.ui, self, db=None) return # we process attachments later del new_doc['_attachments'] if 'signatures' in new_doc['couchapp']: del new_doc['couchapp']['signatures'] for key, value in self.ui.conf.items(): if key == "env": continue elif key == "length": continue elif key == "manifest": continue elif key == "objects": continue elif key == "signatures": continue elif key not in new_doc['couchapp']: new_doc['couchapp'][key] = value for db in self.ui.get_db(dbstring): if self.ui.verbose >= 1: self.ui.logger.info("Pushing CouchApp in %s to design doc:\n%s/%s" % (self.app_dir, db.resource.uri, docid)) index_url = self.index_url(db.resource.uri, app_name, attach_dir, index) if docid in db: design = db[docid] _app_meta = design.get('couchapp', {}) new_doc['couchapp'] ['signatures'] = _app_meta.get('signatures', {}) new_doc.update({ '_rev': design['_rev'], '_attachments': design.get('_attachments', {}) }) else: new_doc.update({'_attachments': {}}) if not kwargs.get('atomic', False): db[docid] = new_doc self.send_attachments(db, design_doc) else: self.encode_attachments(db, design_doc, new_doc) db[docid] = new_doc # send docs maybe we should do bullk update here for doc in docs: new_doc = copy.deepcopy(doc) inline_attachments = {} if isinstance(doc.get('_attachments', False), dict): first_attachment = doc['_attachments'].values()[0] if isinstance(first_attachment, dict): inline_attachments = doc['_attachments'] docid = new_doc['_id'] if docid in db: old_doc = db[docid] doc_meta = old_doc.get('couchapp', {}) doc['couchapp']['signatures'] = doc_meta.get('signatures', {}) new_doc.update({ '_rev': old_doc['_rev'], '_attachments': old_doc.get('_attachments', inline_attachments) }) else: new_doc['couchapp']['signatures'] = {} new_doc.update({'_attachments': inline_attachments}) if not kwargs.get('atomic', False): db[docid] = new_doc if not inline_attachments: self.send_attachments(db, doc) else: if not inline_attachments: self.encode_attachments(db, doc, new_doc) db[docid] = new_doc self.extensions.notify("post-push", self.ui, self, db=db) if index_url: self.ui.logger.info("Visit your CouchApp here:\n%s" % index_url)