def write(self, fname, content): """ write content in a file :attr fname: string,filename :attr content: string """ f = open(fname, 'wb') f.write(utils.to_bytestring(content)) f.close()
def _send_body_part(data, connection): if isinstance(data, types.StringTypes): data = StringIO.StringIO(to_bytestring(data)) elif not hasattr(data, 'read'): data = StringIO.StringIO(str(data)) # we always stream while 1: binarydata = data.read(16384) if binarydata == '': break connection.send(binarydata)
def apply_lib(doc, funcs, app_dir, objs, ui): for k, v in funcs.iteritems(): if not isinstance(v, basestring): continue old_v = v try: funcs[k] = run_json_macros(doc, run_code_macros(v, app_dir, ui), app_dir, ui) except ValueError, e: raise MacroError("Error running !code or !json on function \"%s\": %s" % (k, e)) if old_v != funcs[k]: objs[md5(to_bytestring(funcs[k])).hexdigest()] = old_v
def apply_lib(doc, funcs, app_dir, objs, ui): for k, v in funcs.iteritems(): if not isinstance(v, basestring): continue else: if ui.verbose >= 2: ui.logger.info("process function: %s" % k) old_v = v try: funcs[k] = run_json_macros(doc, run_code_macros(v, app_dir, ui), app_dir, ui) except ValueError, e: raise MacroError('Error running !code or !json on function "%s": %s' % (k, e)) if old_v != funcs[k]: objs[md5(to_bytestring(funcs[k])).hexdigest()] = old_v
def apply_lib(self, funcs, app_dir, verbose=False): if not hasattr(self, "objects"): self.objects = {} for k, v in funcs.iteritems(): if not isinstance(v, basestring): continue old_v = v try: funcs[k] = self.run_json_macros( self.run_code_macros(v, app_dir, verbose=verbose), app_dir, verbose=verbose) except ValueError, e: print >>sys.stderr, "Error running !code or !json on function \"%s\": %s" % (k, e) sys.exit(-1) if old_v != funcs[k]: self.objects[_md5(to_bytestring(funcs[k])).hexdigest()] = old_v
def clone(ui, source, dest=None, rev=None): """ Clone an application from a design_doc given. :attr design_doc: dict, the design doc retrieved from couchdb if something was wrong. """ try: dburl, docid = source.split('_design/') except ValueError: raise AppError("%s isn't a valid source" % source) if not dest: dest = docid path = os.path.normpath(os.path.join(os.getcwd(), dest)) if not os.path.exists(path): os.makedirs(path) db = client.Database(ui, dburl[:-1]) if not rev: doc = db.open_doc("_design/%s" % docid) else: doc = db.open_doc("_design/%s" % docid, rev=rev) docid = doc['_id'] metadata = doc.get('couchapp', {}) # get manifest manifest = metadata.get('manifest', {}) # get signatures signatures = metadata.get('signatures', {}) # get objects refs objects = metadata.get('objects', {}) # create files from manifest if manifest: for filename in manifest: if ui.verbose >=2: ui.logger.info("clone property: %s" % filename) filepath = os.path.join(path, filename) if filename.endswith('/'): if not os.path.isdir(filepath): os.makedirs(filepath) elif filename == "couchapp.json": continue else: parts = ui.split_path(filename) fname = parts.pop() v = doc while 1: try: for key in parts: v = v[key] except KeyError: break # remove extension last_key, ext = os.path.splitext(fname) # make sure key exist try: content = v[last_key] except KeyError: break if isinstance(content, basestring): _ref = md5(to_bytestring(content)).hexdigest() if objects and _ref in objects: content = objects[_ref] if content.startswith('base64-encoded;'): content = base64.b64decode(content[15:]) if fname.endswith('.json'): content = json.dumps(content).encode('utf-8') del v[last_key] # make sure file dir have been created filedir = os.path.dirname(filepath) if not os.path.isdir(filedir): os.makedirs(filedir) ui.write(filepath, content) # remove the key from design doc temp = doc for key2 in parts: if key2 == key: if not temp[key2]: del temp[key2] break temp = temp[key2] # second pass for missing key or in case # manifest isn't in app for key in doc.iterkeys(): if key.startswith('_'): continue elif key in ('couchapp'): app_meta = copy.deepcopy(doc['couchapp']) if 'signatures' in app_meta: del app_meta['signatures'] if 'manifest' in app_meta: del app_meta['manifest'] if 'objects' in app_meta: del app_meta['objects'] if 'length' in app_meta: del app_meta['length'] if app_meta: couchapp_file = os.path.join(path, 'couchapp.json') ui.write_json(couchapp_file, app_meta) elif key in ('views'): vs_dir = os.path.join(path, key) if not os.path.isdir(vs_dir): os.makedirs(vs_dir) for vsname, vs_item in doc[key].iteritems(): vs_item_dir = os.path.join(vs_dir, vsname) if not os.path.isdir(vs_item_dir): os.makedirs(vs_item_dir) for func_name, func in vs_item.iteritems(): filename = os.path.join(vs_item_dir, '%s.js' % func_name) ui.write(filename, func) if ui.verbose >=2: ui.logger.info( "clone view not in manifest: %s" % filename) elif key in ('shows', 'lists', 'filter', 'update'): showpath = os.path.join(path, key) if not os.path.isdir(showpath): os.makedirs(showpath) for func_name, func in doc[key].iteritems(): filename = os.path.join(showpath, '%s.js' % func_name) ui.write(filename, func) if ui.verbose >=2: ui.logger.info( "clone show or list not in manifest: %s" % filename) else: filedir = os.path.join(path, key) if os.path.exists(filedir): continue else: if ui.verbose >=2: ui.logger.info("clone property not in manifest: %s" % key) if isinstance(doc[key], (list, tuple,)): ui.write_json(filedir + ".json", doc[key]) elif isinstance(doc[key], dict): if not os.path.isdir(filedir): os.makedirs(filedir) for field, value in doc[key].iteritems(): fieldpath = os.path.join(filedir, field) if isinstance(value, basestring): if value.startswith('base64-encoded;'): value = base64.b64decode(content[15:]) ui.write(fieldpath, value) else: ui.write_json(fieldpath + '.json', value) else: value = doc[key] if not isinstance(value, basestring): value = str(value) ui.write(filedir, value) # save id idfile = os.path.join(path, '_id') ui.write(idfile, doc['_id']) ui.write_json(os.path.join(path, '.couchapprc'), {}) if '_attachments' in doc: # process attachments attachdir = os.path.join(path, '_attachments') if not os.path.isdir(attachdir): os.makedirs(attachdir) for filename in doc['_attachments'].iterkeys(): if filename.startswith('vendor'): attach_parts = ui.split_path(filename) vendor_attachdir = os.path.join(path, attach_parts.pop(0), attach_parts.pop(0), '_attachments') filepath = os.path.join(vendor_attachdir, *attach_parts) else: filepath = os.path.join(attachdir, filename) filepath = _replace_slash(filepath) currentdir = os.path.dirname(filepath) if not os.path.isdir(currentdir): os.makedirs(currentdir) if signatures.get(filename) != ui.sign(filepath): resp = db.fetch_attachment(docid, filename) with open(filepath, 'wb') as f: for chunk in resp.body_file: f.write(chunk) if ui.verbose>=2: ui.logger.info("clone attachment: %s" % filename)
def request(self, method, path=None, payload=None, headers=None, **params): stream = params.pop('_stream', False) raw_json = params.pop('_raw_json', False) headers = headers or {} headers.update(self._headers.copy()) size = headers.get('Content-Length', None) if payload: if isinstance(payload, file): try: payload.flush() except IOError: pass size = int(os.fstat(payload.fileno())[6]) elif isinstance(payload, types.StringTypes): payload = to_bytestring(payload) size = len(payload) elif not hasattr(payload, 'read') and not isinstance(payload, basestring): payload = json.dumps(payload).encode('utf-8') headers.setdefault('Content-Type', 'application/json') size = len(payload) if payload is not None and size is not None: headers.setdefault('Content-Length', size) if 'Content-Type' not in headers: type_ = None if hasattr(payload, 'name'): type_ = mimetypes.guess_type(payload.name)[0] headers['Content-Type'] = type_ and type_ or 'application/octet-stream' headers = _normalize_headers(headers) uri = url_parser(make_uri(self.url, path, **encode_params(params))) resp, connection = self._request(method, uri, payload=payload, headers=headers) status_code = int(resp.status) self.response = HTTPResponse(resp) if status_code >= 400: data = resp.read() self._release_connection(uri, connection) if status_code == 404: raise ResourceNotFound(data) elif status_code == 401 or status_code == 403: raise Unauthorized(data) elif status_code == 409: raise ResourceConflict(data) elif status_code == 412: raise PreconditionFailed(data) else: raise RequestFailed("Error %s: %s" % (status_code, data)) if raw_json: if stream: return ResponseStream(resp, uri, connection, self._release_connection) data = resp.read() self._release_connection(uri, connection) return _utf8(data) elif stream: return ResponseStream(resp, uri, connection, self._release_connection) data = resp.read() self._release_connection(uri, connection) data = _utf8(data) if resp.getheader('content-type') == 'application/json': try: data = json.loads(data) except ValueError: self.ui.logger.error("can't deserialize response on %s" % uri.geturl()) pass return data
def clone(cls, app_uri, app_dir, verbose=False): """Clone a CouchApp from app_uri into app_dir""" server_uri, db_name, docid = parse_uri(app_uri) couchdb_server = _server(server_uri) try: db = couchdb_server.create(db_name) except: # db already exist db = couchdb_server[db_name] app_name = get_appname(docid) if verbose >= 1: print "Cloning %s to %s..." % (app_name, app_dir) if not app_dir: app_dir = os.path.normpath(os.path.join(os.getcwd(), app_name)) rc_file = os.path.join(app_dir, '.couchapprc') if not os.path.isdir(app_dir): os.makedirs(app_dir) else: # delete only if there is .couchapp folder if os.path.isfile(rc_file): for root, dirs, files in os.walk(app_dir, topdown=False): if root == app_dir: if '_attachments' in dirs: dirs.remove('_attachments') if '.couchapprc' in files: files.remove('.couchapprc') for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) try: design = db[docid] except ResourceNotFound: print >>sys.stderr, "%s don't exist" % app_name return metadata = design.get('couchapp', {}) # get manifest manifest = metadata.get('manifest', {}) # get signatures signatures = metadata.get('signatures', {}) # get objects refs objects = metadata.get('objects', {}) conf = read_json(rc_file) if not 'env' in conf: conf['env'] = {} conf['env'].update({ 'origin': { 'db': db.resource.uri } }) write_json(rc_file, conf) # create files from manifest if manifest: for filename in manifest: if verbose >=2: print "clone property: %s" % filename file_path = os.path.join(app_dir, filename) if filename.endswith('/'): if not os.path.isdir(file_path): os.makedirs(file_path) elif filename == "couchapp.json": continue else: parts = filename.split('/') fname = parts.pop() v = design while 1: try: for key in parts: v = v[key] except KeyError: break # remove extension last_key, ext = os.path.splitext(fname) # make sure key exist try: content = v[last_key] except KeyError: break if isinstance(content, basestring): _ref = _md5(to_bytestring(content)).hexdigest() if objects and _ref in objects: content = objects[_ref] if fname.endswith('.json'): content = json.dumps(content) del v[last_key] # make sure file dir have been created file_dir = os.path.dirname(file_path) if not os.path.isdir(file_dir): os.makedirs(file_dir) write_content(file_path, content) # remove the key from design doc temp = design for key2 in parts: if key2 == key: if not temp[key2]: del temp[key2] break temp = temp[key2] # second pass for missing key or in case # manifest isn't in app for key in design.iterkeys(): if key.startswith('_'): continue elif key in ('couchapp'): app_meta = design['couchapp'].copy() if 'signatures' in app_meta: del app_meta['signatures'] if 'manifest' in app_meta: del app_meta['manifest'] if 'objects' in app_meta: del app_meta['objects'] if app_meta: couchapp_file = os.path.join(app_dir, 'couchapp.json') write_json(couchapp_file, app_meta) elif key in ('views'): vs_dir = os.path.join(app_dir, key) if not os.path.isdir(vs_dir): os.makedirs(vs_dir) for vsname, vs_item in design[key].iteritems(): vs_item_dir = os.path.join(vs_dir, vsname) if not os.path.isdir(vs_item_dir): os.makedirs(vs_item_dir) for func_name, func in vs_item.iteritems(): filename = os.path.join(vs_item_dir, '%s.js' % func_name) open(filename, 'w').write(func) if verbose >=2: print "clone view not in manifest: %s" % filename elif key in ('shows', 'lists'): dir = os.path.join(app_dir, key) if not os.path.isdir(dir): os.makedirs(dir) for func_name, func in design[key].iteritems(): filename = os.path.join(dir, '%s.js' % func_name) open(filename, 'w').write(func) if verbose >=2: print "clone show or list not in manifest: %s" % filename else: file_dir = os.path.join(app_dir, key) if verbose >=2: print "clone property not in manifest: %s" % key if isinstance(design[key], (list, tuple,)): write_json(file_dir + ".json", design[key]) elif isinstance(design[key], dict): if not os.path.isdir(file_dir): os.makedirs(file_dir) for field, value in design[key].iteritems(): field_path = os.path.join(file_dir, field) if isinstance(value, basestring): write_content(field_path, value) else: write_json(field_path + '.json', value) else: value = design[key] if not isinstance(value, basestring): value = str(value) write_content(file_dir, value) # get attachments if '_attachments' in design: attach_dir = os.path.join(app_dir, '_attachments') if not os.path.isdir(attach_dir): os.makedirs(attach_dir) for filename in design['_attachments'].iterkeys(): if filename.startswith('vendor'): attach_parts = filename.split('/') vendor_attach_dir = os.path.join(app_dir, attach_parts.pop(0), attach_parts.pop(0), '_attachments') file_path = os.path.join(vendor_attach_dir, '/'.join(attach_parts)) else: file_path = os.path.join(attach_dir, filename) current_dir = os.path.dirname(file_path) if not os.path.isdir(current_dir): os.makedirs(current_dir) if signatures.get(filename) != sign_file(file_path): content = db.get_attachment(docid, filename) write_content(file_path, content) if verbose>=2: print "clone attachment: %s" % filename