def setup_attachments(self):
'''
Create ``_attachments``
'''
if '_attachments' not in self.doc:
return
attachdir = os.path.join(self.path, '_attachments')
if not os.path.isdir(attachdir):
self.setup_dir(attachdir)
for filename in self.doc['_attachments']:
filepath = os.path.normpath(self.locate_attach_dir(filename))
currentdir = os.path.dirname(filepath)
if not os.path.isdir(currentdir):
self.setup_dir(currentdir)
if self.signatures.get(filename) == util.sign(filepath):
continue # we already have the same file on fs
self.dump_attachment(filename, filepath)
logger.debug('clone attachment: "{0}"'.format(filename))
def clone(source, dest=None, rev=None):
"""
Clone an application from a design_doc given.
:attr design_doc: dict, the design doc retrieved from couchdb
if something was wrong.
"""
try:
dburl, docid = source.split('_design/')
except ValueError:
raise AppError("%s isn't a valid source" % source)
if not dest:
dest = docid
path = os.path.normpath(os.path.join(os.getcwd(), dest))
if not os.path.exists(path):
os.makedirs(path)
db = client.Database(dburl[:-1], create=False)
if not rev:
doc = db.open_doc("_design/%s" % docid)
else:
doc = db.open_doc("_design/%s" % docid, rev=rev)
docid = doc['_id']
metadata = doc.get('couchapp', {})
# get manifest
manifest = metadata.get('manifest', {})
# get signatures
signatures = metadata.get('signatures', {})
# get objects refs
objects = metadata.get('objects', {})
# create files from manifest
if manifest:
for filename in manifest:
logger.debug("clone property: %s" % filename)
filepath = os.path.join(path, filename)
if filename.endswith('/'):
if not os.path.isdir(filepath):
os.makedirs(filepath)
elif filename == "couchapp.json":
continue
else:
parts = util.split_path(filename)
fname = parts.pop()
v = doc
while 1:
try:
for key in parts:
v = v[key]
except KeyError:
break
# remove extension
last_key, ext = os.path.splitext(fname)
# make sure key exist
try:
content = v[last_key]
except KeyError:
break
if isinstance(content, basestring):
_ref = md5(util.to_bytestring(content)).hexdigest()
if objects and _ref in objects:
content = objects[_ref]
if content.startswith('base64-encoded;'):
content = base64.b64decode(content[15:])
if fname.endswith('.json'):
content = util.json.dumps(content).encode('utf-8')
del v[last_key]
# make sure file dir have been created
filedir = os.path.dirname(filepath)
if not os.path.isdir(filedir):
os.makedirs(filedir)
util.write(filepath, content)
# remove the key from design doc
temp = doc
for key2 in parts:
if key2 == key:
if not temp[key2]:
del temp[key2]
break
temp = temp[key2]
# second pass for missing key or in case
# manifest isn't in app
for key in doc.iterkeys():
if key.startswith('_'):
continue
elif key in ('couchapp'):
app_meta = copy.deepcopy(doc['couchapp'])
if 'signatures' in app_meta:
del app_meta['signatures']
if 'manifest' in app_meta:
del app_meta['manifest']
if 'objects' in app_meta:
del app_meta['objects']
if 'length' in app_meta:
del app_meta['length']
if app_meta:
couchapp_file = os.path.join(path, 'couchapp.json')
util.write_json(couchapp_file, app_meta)
elif key in ('views'):
vs_dir = os.path.join(path, key)
if not os.path.isdir(vs_dir):
os.makedirs(vs_dir)
for vsname, vs_item in doc[key].iteritems():
vs_item_dir = os.path.join(vs_dir, vsname)
if not os.path.isdir(vs_item_dir):
os.makedirs(vs_item_dir)
for func_name, func in vs_item.iteritems():
filename = os.path.join(vs_item_dir, '%s.js' % func_name)
util.write(filename, func)
logger.warning("clone view not in manifest: %s" % filename)
elif key in ('shows', 'lists', 'filter', 'updates'):
showpath = os.path.join(path, key)
if not os.path.isdir(showpath):
os.makedirs(showpath)
for func_name, func in doc[key].iteritems():
filename = os.path.join(showpath, '%s.js' % func_name)
util.write(filename, func)
logger.warning("clone show or list not in manifest: %s" %
filename)
else:
filedir = os.path.join(path, key)
if os.path.exists(filedir):
continue
else:
logger.warning("clone property not in manifest: %s" % key)
if isinstance(doc[key], (
list,
tuple,
)):
util.write_json(filedir + ".json", doc[key])
elif isinstance(doc[key], dict):
if not os.path.isdir(filedir):
os.makedirs(filedir)
for field, value in doc[key].iteritems():
fieldpath = os.path.join(filedir, field)
if isinstance(value, basestring):
if value.startswith('base64-encoded;'):
value = base64.b64decode(content[15:])
util.write(fieldpath, value)
else:
util.write_json(fieldpath + '.json', value)
else:
value = doc[key]
if not isinstance(value, basestring):
value = str(value)
util.write(filedir, value)
# save id
idfile = os.path.join(path, '_id')
util.write(idfile, doc['_id'])
util.write_json(os.path.join(path, '.couchapprc'), {})
if '_attachments' in doc: # process attachments
attachdir = os.path.join(path, '_attachments')
if not os.path.isdir(attachdir):
os.makedirs(attachdir)
for filename in doc['_attachments'].iterkeys():
if filename.startswith('vendor'):
attach_parts = util.split_path(filename)
vendor_attachdir = os.path.join(path, attach_parts.pop(0),
attach_parts.pop(0),
'_attachments')
filepath = os.path.join(vendor_attachdir, *attach_parts)
else:
filepath = os.path.join(attachdir, filename)
filepath = _replace_slash(filepath)
currentdir = os.path.dirname(filepath)
if not os.path.isdir(currentdir):
os.makedirs(currentdir)
if signatures.get(filename) != util.sign(filepath):
resp = db.fetch_attachment(docid, filename)
with open(filepath, 'wb') as f:
for chunk in resp.body_stream():
f.write(chunk)
logger.debug("clone attachment: %s" % filename)
logger.info("%s cloned in %s" % (source, dest))
def doc(self, db=None, with_attachments=True, force=False):
""" Function to reetrieve document object from
document directory. If `with_attachments` is True
attachments will be included and encoded"""
manifest = []
objects = {}
signatures = {}
attachments = {}
self._doc = {"_id": self.docid}
# get designdoc
self._doc.update(self.dir_to_fields(self.docdir, manifest=manifest))
if not "couchapp" in self._doc:
self._doc["couchapp"] = {}
self.olddoc = {}
if db is not None:
try:
self.olddoc = db.open_doc(self._doc["_id"])
attachments = self.olddoc.get("_attachments") or {}
self._doc.update({"_rev": self.olddoc["_rev"]})
except ResourceNotFound:
self.olddoc = {}
if "couchapp" in self.olddoc:
old_signatures = self.olddoc["couchapp"].get("signatures", {})
else:
old_signatures = {}
for name, filepath in self.attachments():
signatures[name] = util.sign(filepath)
if with_attachments and not old_signatures:
logger.debug("attach %s ", name)
attachments[name] = self.attachment_stub(name, filepath)
if old_signatures:
for name, signature in old_signatures.items():
cursign = signatures.get(name)
if not cursign:
logger.debug("detach %s ", name)
del attachments[name]
elif cursign != signature:
logger.debug("detach %s ", name)
del attachments[name]
else:
continue
if with_attachments:
for name, filepath in self.attachments():
if old_signatures.get(name) != signatures.get(name) or force:
logger.debug("attach %s ", name)
attachments[name] = self.attachment_stub(name, filepath)
self._doc["_attachments"] = attachments
self._doc["couchapp"].update({"manifest": manifest, "objects": objects, "signatures": signatures})
if self.docid.startswith("_design/"): # process macros
for funs in ["shows", "lists", "updates", "filters", "spatial"]:
if funs in self._doc:
package_shows(self._doc, self._doc[funs], self.docdir, objects)
if "validate_doc_update" in self._doc:
tmp_dict = {"validate_doc_update": self._doc["validate_doc_update"]}
package_shows(self._doc, tmp_dict, self.docdir, objects)
self._doc.update(tmp_dict)
if "views" in self._doc:
# clean views
# we remove empty views and malformed from the list
# of pushed views. We also clean manifest
views = {}
dmanifest = {}
for i, fname in enumerate(manifest):
if fname.startswith("views/") and fname != "views/":
name, ext = os.path.splitext(fname)
if name.endswith("/"):
name = name[:-1]
dmanifest[name] = i
for vname, value in self._doc["views"].iteritems():
if value and isinstance(value, dict):
views[vname] = value
else:
del manifest[dmanifest["views/%s" % vname]]
self._doc["views"] = views
package_views(self._doc, self._doc["views"], self.docdir, objects)
if "fulltext" in self._doc:
package_views(self._doc, self._doc["fulltext"], self.docdir, objects)
return self._doc
def clone(source, dest=None, rev=None):
"""
Clone an application from a design_doc given.
:attr design_doc: dict, the design doc retrieved from couchdb
if something was wrong.
"""
try:
dburl, docid = source.split('_design/')
except ValueError:
raise AppError("%s isn't a valid source" % source)
if not dest:
dest = docid
path = os.path.normpath(os.path.join(os.getcwd(), dest))
if not os.path.exists(path):
os.makedirs(path)
db = client.Database(dburl[:-1], create=False)
if not rev:
doc = db.open_doc("_design/%s" % docid)
else:
doc = db.open_doc("_design/%s" % docid, rev=rev)
docid = doc['_id']
metadata = doc.get('couchapp', {})
# get manifest
manifest = metadata.get('manifest', {})
# get signatures
signatures = metadata.get('signatures', {})
# get objects refs
objects = metadata.get('objects', {})
# create files from manifest
if manifest:
for filename in manifest:
logger.debug("clone property: %s" % filename)
filepath = os.path.join(path, filename)
if filename.endswith('/'):
if not os.path.isdir(filepath):
os.makedirs(filepath)
elif filename == "couchapp.json":
continue
else:
parts = util.split_path(filename)
fname = parts.pop()
v = doc
while 1:
try:
for key in parts:
v = v[key]
except KeyError:
break
# remove extension
last_key, ext = os.path.splitext(fname)
# make sure key exist
try:
content = v[last_key]
except KeyError:
break
if isinstance(content, basestring):
_ref = md5(util.to_bytestring(content)).hexdigest()
if objects and _ref in objects:
content = objects[_ref]
if content.startswith('base64-encoded;'):
content = base64.b64decode(content[15:])
if fname.endswith('.json'):
content = util.json.dumps(content).encode('utf-8')
del v[last_key]
# make sure file dir have been created
filedir = os.path.dirname(filepath)
if not os.path.isdir(filedir):
os.makedirs(filedir)
util.write(filepath, content)
# remove the key from design doc
temp = doc
for key2 in parts:
if key2 == key:
if not temp[key2]:
del temp[key2]
break
temp = temp[key2]
# second pass for missing key or in case
# manifest isn't in app
for key in doc.iterkeys():
if key.startswith('_'):
continue
elif key in ('couchapp'):
app_meta = copy.deepcopy(doc['couchapp'])
if 'signatures' in app_meta:
del app_meta['signatures']
if 'manifest' in app_meta:
del app_meta['manifest']
if 'objects' in app_meta:
del app_meta['objects']
if 'length' in app_meta:
del app_meta['length']
if app_meta:
couchapp_file = os.path.join(path, 'couchapp.json')
util.write_json(couchapp_file, app_meta)
elif key in ('views'):
vs_dir = os.path.join(path, key)
if not os.path.isdir(vs_dir):
os.makedirs(vs_dir)
for vsname, vs_item in doc[key].iteritems():
vs_item_dir = os.path.join(vs_dir, vsname)
if not os.path.isdir(vs_item_dir):
os.makedirs(vs_item_dir)
for func_name, func in vs_item.iteritems():
filename = os.path.join(vs_item_dir, '%s.js' % func_name)
util.write(filename, func)
logger.warning("clone view not in manifest: %s" % filename)
elif key in ('shows', 'lists', 'filter', 'update'):
showpath = os.path.join(path, key)
if not os.path.isdir(showpath):
os.makedirs(showpath)
for func_name, func in doc[key].iteritems():
filename = os.path.join(showpath, '%s.js' % func_name)
util.write(filename, func)
logger.warning(
"clone show or list not in manifest: %s" % filename)
else:
filedir = os.path.join(path, key)
if os.path.exists(filedir):
continue
else:
logger.warning("clone property not in manifest: %s" % key)
if isinstance(doc[key], (list, tuple,)):
util.write_json(filedir + ".json", doc[key])
elif isinstance(doc[key], dict):
if not os.path.isdir(filedir):
os.makedirs(filedir)
for field, value in doc[key].iteritems():
fieldpath = os.path.join(filedir, field)
if isinstance(value, basestring):
if value.startswith('base64-encoded;'):
value = base64.b64decode(content[15:])
util.write(fieldpath, value)
else:
util.write_json(fieldpath + '.json', value)
else:
value = doc[key]
if not isinstance(value, basestring):
value = str(value)
util.write(filedir, value)
# save id
idfile = os.path.join(path, '_id')
util.write(idfile, doc['_id'])
util.write_json(os.path.join(path, '.couchapprc'), {})
if '_attachments' in doc: # process attachments
attachdir = os.path.join(path, '_attachments')
if not os.path.isdir(attachdir):
os.makedirs(attachdir)
for filename in doc['_attachments'].iterkeys():
if filename.startswith('vendor'):
attach_parts = util.split_path(filename)
vendor_attachdir = os.path.join(path, attach_parts.pop(0),
attach_parts.pop(0),
'_attachments')
filepath = os.path.join(vendor_attachdir, *attach_parts)
else:
filepath = os.path.join(attachdir, filename)
filepath = _replace_slash(filepath)
currentdir = os.path.dirname(filepath)
if not os.path.isdir(currentdir):
os.makedirs(currentdir)
if signatures.get(filename) != util.sign(filepath):
resp = db.fetch_attachment(docid, filename)
with open(filepath, 'wb') as f:
for chunk in resp.body_stream():
f.write(chunk)
logger.debug("clone attachment: %s" % filename)
logger.info("%s cloned in %s" % (source, dest))
def doc(self, db=None, with_attachments=True):
""" Function to reetrieve document object from
document directory. If `with_attachments` is True
attachments will be included and encoded"""
manifest = []
objects = {}
self._doc = {'_id': self.docid}
# get designdoc
self._doc.update(self.dir_to_fields(self.docdir, manifest=manifest))
if not 'couchapp' in self._doc:
self._doc['couchapp'] = {}
signatures = {}
attachments = {}
for name, filepath in self.attachments():
signatures[name] = util.sign(filepath)
if with_attachments:
logger.debug("attach %s " % name)
attachments[name] = {}
with open(filepath, "rb") as f:
re_sp = re.compile('\s')
attachments[name]['data'] = re_sp.sub('',
base64.b64encode(f.read()))
attachments[name]['content_type'] = ';'.join(filter(None,
mimetypes.guess_type(name)))
if with_attachments:
self._doc['_attachments'] = attachments
self._doc['couchapp'].update({
'manifest': manifest,
'objects': objects,
'signatures': signatures
})
if self.docid.startswith('_design/'): # process macros
for funs in ['shows', 'lists', 'updates', 'filters',
'fulltext']:
if funs in self._doc:
package_shows(self._doc, self._doc[funs], self.docdir,
objects)
if 'validate_doc_update' in self._doc:
tmp_dict = dict(validate_doc_update=self._doc[
"validate_doc_update"])
package_shows( self._doc, tmp_dict, self.docdir,
objects)
self._doc.update(tmp_dict)
if 'views' in self._doc:
# clean views
# we remove empty views and malformed from the list
# of pushed views. We also clean manifest
views = {}
dmanifest = {}
for i, fname in enumerate(manifest):
if fname.startswith("views/") and fname != "views/":
name, ext = os.path.splitext(fname)
if name.endswith('/'):
name = name[:-1]
dmanifest[name] = i
for vname, value in self._doc['views'].iteritems():
if value and isinstance(value, dict):
views[vname] = value
else:
del manifest[dmanifest["views/%s" % vname]]
self._doc['views'] = views
package_views(self._doc,self._doc["views"], self.docdir,
objects)
self.olddoc = {}
if db is not None:
try:
self.olddoc = db.open_doc(self._doc['_id'])
self._doc.update({'_rev': self.olddoc['_rev']})
except ResourceNotFound:
pass
return self._doc
def doc(self, db=None, with_attachments=True, force=False):
"""
Function to retrieve document object from document directory.
:param with_attachments: If ``True``,
attachments will be included and encoded
"""
manifest = []
objects = {}
signatures = {}
attachments = {}
self._doc = {'_id': self.docid}
# get designdoc
self._doc.update(self.dir_to_fields(self.docdir, manifest=manifest))
if not 'couchapp' in self._doc:
self._doc['couchapp'] = {}
self.olddoc = {}
if db is not None:
try:
self.olddoc = db.open_doc(self._doc['_id'])
attachments = self.olddoc.get('_attachments') or {}
self._doc.update({'_rev': self.olddoc['_rev']})
except ResourceNotFound:
self.olddoc = {}
if 'couchapp' in self.olddoc:
old_signatures = self.olddoc['couchapp'].get('signatures', {})
else:
old_signatures = {}
for name, filepath in self.attachments():
signatures[name] = util.sign(filepath)
if with_attachments and not old_signatures:
logger.debug("attach %s ", name)
attachments[name] = self.attachment_stub(name, filepath)
if old_signatures:
for name, signature in old_signatures.items():
cursign = signatures.get(name)
if not cursign:
logger.debug("detach %s ", name)
del attachments[name]
elif cursign != signature:
logger.debug("detach %s ", name)
del attachments[name]
else:
continue
if with_attachments:
for name, filepath in self.attachments():
if old_signatures.get(name) != \
signatures.get(name) or force:
logger.debug("attach %s ", name)
attachments[name] = self.attachment_stub(
name, filepath)
self._doc['_attachments'] = attachments
self._doc['couchapp'].update({
'manifest': manifest,
'objects': objects,
'signatures': signatures
})
if self.docid.startswith('_design/'): # process macros
for funs in ['shows', 'lists', 'updates', 'filters', 'spatial']:
if funs in self._doc:
package_shows(self._doc, self._doc[funs], self.docdir,
objects)
if 'validate_doc_update' in self._doc:
tmp_dict = {
'validate_doc_update': self._doc["validate_doc_update"]
}
package_shows(self._doc, tmp_dict, self.docdir, objects)
self._doc.update(tmp_dict)
if 'views' in self._doc:
# clean views
# we remove empty views and malformed from the list
# of pushed views. We also clean manifest
views = {}
dmanifest = {}
for i, fname in enumerate(manifest):
if fname.startswith("views/") and fname != "views/":
name, ext = os.path.splitext(fname)
if name.endswith('/'):
name = name[:-1]
dmanifest[name] = i
for vname, value in self._doc['views'].iteritems():
if value and isinstance(value, dict):
views[vname] = value
else:
del manifest[dmanifest["views/%s" % vname]]
self._doc['views'] = views
package_views(self._doc, self._doc["views"], self.docdir,
objects)
if "fulltext" in self._doc:
package_views(self._doc, self._doc["fulltext"], self.docdir,
objects)
return self._doc