def _encode_options(self, options):
retval = {}
for name, value in options.items():
if name in ('key', 'startkey', 'endkey') \
or not isinstance(value, basestring):
value = json.dumps(value, allow_nan=False, ensure_ascii=False)
retval[name] = value
return retval
def write_json(self, fname, content):
""" serialize content in json and save it
:attr fname: string
:attr content: string
"""
self.write(fname, json.dumps(content))
def _exec(self, options):
body = {'map': self.map_fun, 'language': self.language}
if self.reduce_fun:
body['reduce'] = self.reduce_fun
if 'keys' in options:
options = options.copy()
body['keys'] = options.pop('keys')
content = json.dumps(body, allow_nan=False,
ensure_ascii=False).encode('utf-8')
resp, data = self.resource.post(content=content, headers={
'Content-Type': 'application/json'
}, **self._encode_options(options))
return data
def _request(self, method, path=None, content=None, headers=None,
**params):
from couchapp import __version__
headers = headers or {}
headers.setdefault('Accept', 'application/json')
headers.setdefault('User-Agent', 'couchapp %s' % __version__)
body = None
if content is not None:
if not isinstance(content, basestring):
body = json.dumps(content, allow_nan=False,
ensure_ascii=False).encode('utf-8')
headers.setdefault('Content-Type', 'application/json')
else:
body = content
headers.setdefault('Content-Length', str(len(body)))
def _make_request(retry=1):
try:
return self.http.request(uri(self.uri, path, **params), method,
body=body, headers=headers)
except socket.error, e:
if retry > 0 and e.args[0] == 54: # reset by peer
return _make_request(retry - 1)
raise
if not filenum:
raise MacroError("Processing code: No file matching '%s'" % mo.group(2))
else:
fields = mo.group(2).split('.')
library = doc
count = len(fields)
include_to = included
for i, field in enumerate(fields):
if not field in library: break
library = library[field]
if i+1 < count:
include_to[field] = include_to.get(field, {})
include_to = include_to[field]
else:
include_to[field] = library
return f_string
def rjson2(mo):
return '\n'.join(varstrings)
re_json = re.compile('(\/\/|#)\ ?!json (.*)')
re_json.sub(rjson, f_string)
if not included:
return f_string
for k, v in included.iteritems():
varstrings.append("var %s = %s;" % (k, json.dumps(v)))
return re_json.sub(rjson2, f_string)
def designdoc_to_fs(self, db, design_doc):
"""
Clone an application from a design_doc given.
:attr design_doc: dict, the design doc retrieved from couchdb
if something was wrong.
"""
app_name = get_appname(design_doc["_id"])
docid = design_doc["_id"]
metadata = design_doc.get("couchapp", {})
# get manifest
manifest = metadata.get("manifest", {})
# get signatures
signatures = metadata.get("signatures", {})
# get objects refs
objects = metadata.get("objects", {})
# create files from manifest
if manifest:
for filename in manifest:
if self.ui.verbose >= 2:
print "clone property: %s" % filename
file_path = self.ui.rjoin(self.app_dir, filename)
if filename.endswith("/"):
if not self.ui.isdir(file_path):
self.ui.makedirs(file_path)
elif filename == "couchapp.json":
continue
else:
parts = filename.split("/")
fname = parts.pop()
v = design_doc
while 1:
try:
for key in parts:
v = v[key]
except KeyError:
break
# remove extension
last_key, ext = os.path.splitext(fname)
# make sure key exist
try:
content = v[last_key]
except KeyError:
break
if isinstance(content, basestring):
_ref = md5(to_bytestring(content)).hexdigest()
if objects and _ref in objects:
content = objects[_ref]
if fname.endswith(".json"):
content = json.dumps(content)
del v[last_key]
# make sure file dir have been created
file_dir = self.ui.dirname(file_path)
if not self.ui.isdir(file_dir):
self.ui.makedirs(file_dir)
self.ui.write(file_path, content)
# remove the key from design doc
temp = design_doc
for key2 in parts:
if key2 == key:
if not temp[key2]:
del temp[key2]
break
temp = temp[key2]
# second pass for missing key or in case
# manifest isn't in app
for key in design_doc.iterkeys():
if key.startswith("_"):
continue
elif key in ("couchapp"):
app_meta = copy.deepcopy(design_doc["couchapp"])
if "signatures" in app_meta:
del app_meta["signatures"]
if "manifest" in app_meta:
del app_meta["manifest"]
if "objects" in app_meta:
del app_meta["objects"]
if "lenght" in app_meta:
del app_meta["lenght"]
if app_meta:
couchapp_file = self.ui.rjoin(self.app_dir, "couchapp.json")
self.ui.write_json(couchapp_file, app_meta)
elif key in ("views"):
vs_dir = self.ui.rjoin(self.app_dir, key)
if not self.ui.isdir(vs_dir):
self.ui.makedirs(vs_dir)
for vsname, vs_item in design_doc[key].iteritems():
vs_item_dir = self.ui.rjoin(vs_dir, vsname)
if not self.ui.isdir(vs_item_dir):
self.ui.makedirs(vs_item_dir)
for func_name, func in vs_item.iteritems():
filename = self.ui.rjoin(vs_item_dir, "%s.js" % func_name)
self.ui.write(filename, func)
if self.ui.verbose >= 2:
self.ui.logger.info("clone view not in manifest: %s" % filename)
elif key in ("shows", "lists"):
dir = self.ui.rjoin(self.app_dir, key)
if not self.ui.isdir(dir):
self.ui.makedirs(dir)
for func_name, func in design_doc[key].iteritems():
filename = self.ui.rjoin(dir, "%s.js" % func_name)
self.ui.write(filename, func)
if self.ui.verbose >= 2:
self.ui.logger.info("clone show or list not in manifest: %s" % filename)
else:
file_dir = self.ui.rjoin(self.app_dir, key)
if self.ui.exists(file_dir):
continue
else:
if self.ui.verbose >= 2:
self.ui.logger.info("clone property not in manifest: %s" % key)
if isinstance(design_doc[key], (list, tuple)):
self.ui.write_json(file_dir + ".json", design[key])
elif isinstance(design_doc[key], dict):
if not self.ui.isdir(file_dir):
self.ui.makedirs(file_dir)
for field, value in design_doc[key].iteritems():
field_path = self.ui.rjoin(file_dir, field)
if isinstance(value, basestring):
self.ui.write(field_path, value)
else:
self.ui.write_json(field_path + ".json", value)
else:
value = design_doc[key]
if not isinstance(value, basestring):
value = str(value)
self.ui.write(file_dir, value)
# get attachments
if "_attachments" in design_doc:
attach_dir = self.ui.rjoin(self.app_dir, "_attachments")
if not self.ui.isdir(attach_dir):
self.ui.makedirs(attach_dir)
for filename in design_doc["_attachments"].iterkeys():
if filename.startswith("vendor"):
attach_parts = filename.split("/")
vendor_attach_dir = self.ui.rjoin(
self.app_dir, attach_parts.pop(0), attach_parts.pop(0), "_attachments"
)
file_path = self.ui.rjoin(vendor_attach_dir, "/".join(attach_parts))
else:
file_path = self.ui.rjoin(attach_dir, filename)
current_dir = self.ui.dirname(file_path)
if not self.ui.isdir(current_dir):
self.ui.makedirs(current_dir)
if signatures.get(filename) != self.ui.sign(file_path):
self.ui.write(file_path, db.get_attachment(docid, filename))
if self.ui.verbose >= 2:
self.ui.logger.info("clone attachment: %s" % filename)
def designdoc_to_fs(self, db, design_doc):
"""
Clone an application from a design_doc given.
:attr design_doc: dict, the design doc retrieved from couchdb
if something was wrong.
"""
app_name = get_appname(design_doc['_id'])
docid = design_doc['_id']
metadata = design_doc.get('couchapp', {})
# get manifest
manifest = metadata.get('manifest', {})
# get signatures
signatures = metadata.get('signatures', {})
# get objects refs
objects = metadata.get('objects', {})
# create files from manifest
if manifest:
for filename in manifest:
if self.ui.verbose >=2:
self.ui.logger.info("clone property: %s" % filename)
file_path = self.ui.rjoin(self.app_dir, filename)
if filename.endswith('/'):
if not self.ui.isdir(file_path):
self.ui.makedirs(file_path)
elif filename == "couchapp.json":
continue
else:
parts = self.ui.split_path(filename)
fname = parts.pop()
v = design_doc
while 1:
try:
for key in parts:
v = v[key]
except KeyError:
break
# remove extension
last_key, ext = os.path.splitext(fname)
# make sure key exist
try:
content = v[last_key]
except KeyError:
break
if isinstance(content, basestring):
_ref = md5(to_bytestring(content)).hexdigest()
if objects and _ref in objects:
content = objects[_ref]
if content.startswith('base64-encoded;'):
content = base64.b64decode(content[15:])
if fname.endswith('.json'):
content = json.dumps(content)
del v[last_key]
# make sure file dir have been created
file_dir = self.ui.dirname(file_path)
if not self.ui.isdir(file_dir):
self.ui.makedirs(file_dir)
self.ui.write(file_path, content)
# remove the key from design doc
temp = design_doc
for key2 in parts:
if key2 == key:
if not temp[key2]:
del temp[key2]
break
temp = temp[key2]
# second pass for missing key or in case
# manifest isn't in app
for key in design_doc.iterkeys():
if key.startswith('_'):
continue
elif key in ('couchapp'):
app_meta = copy.deepcopy(design_doc['couchapp'])
if 'signatures' in app_meta:
del app_meta['signatures']
if 'manifest' in app_meta:
del app_meta['manifest']
if 'objects' in app_meta:
del app_meta['objects']
if 'lenght' in app_meta:
del app_meta['lenght']
if app_meta:
couchapp_file = self.ui.rjoin(self.app_dir, 'couchapp.json')
self.ui.write_json(couchapp_file, app_meta)
elif key in ('views'):
vs_dir = self.ui.rjoin(self.app_dir, key)
if not self.ui.isdir(vs_dir):
self.ui.makedirs(vs_dir)
for vsname, vs_item in design_doc[key].iteritems():
vs_item_dir = self.ui.rjoin(vs_dir, vsname)
if not self.ui.isdir(vs_item_dir):
self.ui.makedirs(vs_item_dir)
for func_name, func in vs_item.iteritems():
filename = self.ui.rjoin(vs_item_dir, '%s.js' %
func_name)
self.ui.write(filename, func)
if self.ui.verbose >=2:
self.ui.logger.info("clone view not in manifest: %s" % filename)
elif key in ('shows', 'lists'):
show_path = self.ui.rjoin(self.app_dir, key)
if not self.ui.isdir(show_path):
self.ui.makedirs(show_path)
for func_name, func in design_doc[key].iteritems():
filename = self.ui.rjoin(show_path, '%s.js' %
func_name)
self.ui.write(filename, func)
if self.ui.verbose >=2:
self.ui.logger.info("clone show or list not in manifest: %s" % filename)
else:
file_dir = self.ui.rjoin(self.app_dir, key)
if self.ui.exists(file_dir):
continue
else:
if self.ui.verbose >=2:
self.ui.logger.info("clone property not in manifest: %s" % key)
if isinstance(design_doc[key], (list, tuple,)):
self.ui.write_json(file_dir + ".json", design[key])
elif isinstance(design_doc[key], dict):
if not self.ui.isdir(file_dir):
self.ui.makedirs(file_dir)
for field, value in design_doc[key].iteritems():
field_path = self.ui.rjoin(file_dir, field)
if isinstance(value, basestring):
if value.startswith('base64-encoded;'):
value = base64.b64decode(content[15:])
self.ui.write(field_path, value)
else:
self.ui.write_json(field_path + '.json', value)
else:
value = design_doc[key]
if not isinstance(value, basestring):
value = str(value)
self.ui.write(file_dir, value)
# get attachments
if '_attachments' in design_doc:
attach_dir = self.ui.rjoin(self.app_dir, '_attachments')
if not self.ui.isdir(attach_dir):
self.ui.makedirs(attach_dir)
for filename in design_doc['_attachments'].iterkeys():
if filename.startswith('vendor'):
attach_parts = self.ui.split_path(filename)
vendor_attach_dir = self.ui.rjoin(self.app_dir, attach_parts.pop(0),
attach_parts.pop(0), '_attachments')
file_path = self.ui.rjoin(vendor_attach_dir, *attach_parts)
else:
file_path = self.ui.rjoin(attach_dir, filename)
current_dir = self.ui.dirname(file_path)
if not self.ui.isdir(current_dir):
self.ui.makedirs(current_dir)
if signatures.get(filename) != self.ui.sign(file_path):
self.ui.write(file_path, db.get_attachment(docid, filename))
if self.ui.verbose>=2:
self.ui.logger.info("clone attachment: %s" % filename)
def push(self, dbstring, app_name, **kwargs):
"""Pushes the app specified to the CouchDB instance
:attr dbstring: string, db url or db environment name.
:attr app_name: name of app to push.
:attr verbose: boolean, default is False
"""
self.extensions.notify("pre-push", self.ui, self)
app_name = self.ui.get_app_name(dbstring, app_name)
design_doc = self.fs_to_designdoc(app_name)
docid = design_doc['_id']
attach_dir = self.ui.rjoin(self.app_dir, '_attachments')
new_doc = copy.deepcopy(design_doc)
couchapp = design_doc.get('couchapp', {})
if couchapp:
index = couchapp.get('index', False)
else:
index = False
new_doc['couchapp'] = {}
# get docs from _docs folder
docs_dir = self.ui.rjoin(self.app_dir, '_docs')
if self.ui.isdir(docs_dir):
docs = self.fs_to_docs(docs_dir)
else:
docs = []
# do we export ?
if kwargs.get('export', False):
# process attachments
design = copy.deepcopy(design_doc)
del new_doc['_attachments']
if not 'couchapp' in design:
design['couchapp'] = {}
attachments = design['_attachments']
_length = design['couchapp'].get('length', {})
new_attachments = {}
for filename, value in attachments.iteritems():
content_length = _length.get(filename, None)
if self.ui.verbose >= 2:
self.ui.logger.info("Attaching %s (%s)" % (filename, content_length))
f = open(value, "rb")
# fix issue with httplib that raises BadStatusLine
# error because it didn't close the connection
new_attachments[filename] = {
"content_type": ';'.join(filter(None, mimetypes.guess_type(filename))),
"data": base64.b64encode(f.read()),
}
# update signatures
if not 'couchapp' in new_doc:
new_doc['couchapp'] = {}
new_doc['_attachments'] = new_attachments
if kwargs.get('output', None) is not None:
self.ui.write_json(kwargs.get('output'), new_doc)
else:
print json.dumps(new_doc)
self.extensions.notify("post-push", self.ui, self, db=None)
return
# we process attachments later
del new_doc['_attachments']
if 'signatures' in new_doc['couchapp']:
del new_doc['couchapp']['signatures']
for key, value in self.ui.conf.items():
if key == "env":
continue
elif key == "length":
continue
elif key == "manifest":
continue
elif key == "objects":
continue
elif key == "signatures":
continue
elif key not in new_doc['couchapp']:
new_doc['couchapp'][key] = value
for db in self.ui.get_db(dbstring):
if self.ui.verbose >= 1:
self.ui.logger.info("Pushing CouchApp in %s to design doc:\n%s/%s" % (self.app_dir,
db.resource.uri, docid))
index_url = self.index_url(db.resource.uri, app_name, attach_dir, index)
if docid in db:
design = db[docid]
_app_meta = design.get('couchapp', {})
new_doc['couchapp'] ['signatures'] = _app_meta.get('signatures', {})
new_doc.update({
'_rev': design['_rev'],
'_attachments': design.get('_attachments', {})
})
else:
new_doc.update({'_attachments': {}})
if not kwargs.get('atomic', False):
db[docid] = new_doc
self.send_attachments(db, design_doc)
else:
self.encode_attachments(db, design_doc, new_doc)
db[docid] = new_doc
# send docs maybe we should do bullk update here
for doc in docs:
new_doc = copy.deepcopy(doc)
inline_attachments = {}
if isinstance(doc.get('_attachments', False), dict):
first_attachment = doc['_attachments'].values()[0]
if isinstance(first_attachment, dict):
inline_attachments = doc['_attachments']
docid = new_doc['_id']
if docid in db:
old_doc = db[docid]
doc_meta = old_doc.get('couchapp', {})
doc['couchapp']['signatures'] = doc_meta.get('signatures', {})
new_doc.update({
'_rev': old_doc['_rev'],
'_attachments': old_doc.get('_attachments', inline_attachments)
})
else:
new_doc['couchapp']['signatures'] = {}
new_doc.update({'_attachments': inline_attachments})
if not kwargs.get('atomic', False):
db[docid] = new_doc
if not inline_attachments:
self.send_attachments(db, doc)
else:
if not inline_attachments:
self.encode_attachments(db, doc, new_doc)
db[docid] = new_doc
self.extensions.notify("post-push", self.ui, self, db=db)
if index_url:
self.ui.logger.info("Visit your CouchApp here:\n%s" % index_url)
