def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False): db = Database(dburl) if username is not None and password is not None: db.resource.http.add_credentials(username, password) for headers, is_multipart, payload in read_multipart(fileobj): docid = headers['content-id'] if is_multipart: # doc has attachments for headers, _, payload in payload: if 'content-id' not in headers: doc = json.decode(payload) doc['_attachments'] = {} else: doc['_attachments'][headers['content-id']] = { 'data': b64encode(payload), 'content_type': headers['content-type'], 'length': len(payload) } else: # no attachments, just the JSON doc = json.decode(payload) del doc['_rev'] sys.stderr.write('Loading document %r' % docid) try: db[docid] = doc except Exception as e: if not ignore_errors: raise sys.stderr.write('Error: %s' % e)
def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False): db = Database(dburl) if username is not None and password is not None: db.resource.credentials = (username, password) for headers, is_multipart, payload in read_multipart(fileobj): docid = headers['content-id'] if is_multipart: # doc has attachments for headers, _, payload in payload: if 'content-id' not in headers: doc = json.decode(payload) doc['_attachments'] = {} else: doc['_attachments'][headers['content-id']] = { 'data': b64encode(payload), 'content_type': headers['content-type'], 'length': len(payload) } else: # no attachments, just the JSON doc = json.decode(payload) del doc['_rev'] print >> sys.stderr, 'Loading document %r' % docid try: db[docid] = doc except Exception as e: if not ignore_errors: raise print >> sys.stderr, 'Error: %s' % e
def copy(self, src, dest): """Copy the given document to create a new document. :param src: the ID of the document to copy, or a dictionary or `Document` object representing the source document. :param dest: either the destination document ID as string, or a dictionary or `Document` instance of the document that should be overwritten. :return: the new revision of the destination document :rtype: `str` :since: 0.6 """ if not isinstance(src, basestring): if not isinstance(src, dict): if hasattr(src, "items"): src = dict(src.items()) else: raise TypeError("expected dict or string, got %s" % type(src)) src = src["_id"] if not isinstance(dest, basestring): if not isinstance(dest, dict): if hasattr(dest, "items"): dest = dict(dest.items()) else: raise TypeError("expected dict or string, got %s" % type(dest)) if "_rev" in dest: dest = "%s?%s" % (http.quote(dest["_id"]), http.urlencode({"rev": dest["_rev"]})) else: dest = http.quote(dest["_id"]) _, _, data = self.resource._request("COPY", src, headers={"Destination": dest}) data = json.decode(data.read()) return data["rev"]
def load_multipart_file(fn): fh = file(fn, 'r') #codecs.open(fn, 'r', 'utf-8') for headers, is_multipart, payload in read_multipart(fh): doc = cdb_json.decode(payload) #print "Reading speech", doc.get('speech_id'), "..." speech = Speech(doc) yield speech #speeches.append() fh.close()
def get_revisions_dict(database, row): doc = urllib2.urlopen(SERVER + "/" + database + "/" + row['key'] + "?revs_info=true").read() doc = json.decode(doc) result = [] for dictionary in doc.get('_revs_info'): if dictionary.get('status') == "available": result.append(dictionary.get('rev')) return result
def tmp(docIdx): resp, content = http.request('%s/%s/%s' % (params.server, params.testdb, docs[i]), 'GET') obj = json.decode(content) if obj.has_key('counter'): obj['counter'] = obj['counter'] + 1 else: obj['counter'] = 0 doc = json.encode(obj) http.request('%s/%s/%s' % (params.server, params.testdb, docs[i]), 'PUT', body=doc)
def _changes(self, **opts): _, _, data = self.resource.get('_changes', **opts) lines = data.iterchunks() for ln in lines: if not ln: # skip heartbeats continue doc = json.decode(ln.decode('utf-8')) if 'last_seq' in doc: # consume the rest of the response if this for ln in lines: # was the last line, allows conn reuse pass yield doc
def _changes(self, **opts): _, _, data = self.resource.get('_changes', **opts) lines = iter(data) for ln in lines: if not ln: # skip heartbeats continue doc = json.decode(ln) if 'last_seq' in doc: # consume the rest of the response if this for ln in lines: # was the last line, allows conn reuse pass yield doc
def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False): db = Database(dburl) if username is not None and password is not None: db.resource.credentials = (username, password) hostname = Is_Server(dburl, db.name) for headers, is_multipart, payload in read_multipart(fileobj): docid = headers['content-id'] time.sleep(1) if '-db' in db.name: if 'credential' in docid or 'flow' in docid or 'setting' in docid or 'functions' in docid: docid = Rename_docid(docid, hostname) obj = db.get(docid) if obj == None: new_doc = {'_id': docid} db.save(new_doc) obj = db.get(docid) if is_multipart: # doc has attachments for headers, _, payload in payload: if 'content-id' not in headers: doc = json.decode(payload) doc['_attachments'] = {} else: doc['_attachments'][headers['content-id']] = { 'data': b64encode(payload).decode('ascii'), 'content_type': headers['content-type'], 'length': len(payload) } else: # no attachments, just the JSON doc = json.decode(payload) doc['_rev'] = obj['_rev'] doc['_id'] = obj['_id'] print('Loading document %r' % docid, file=sys.stderr) try: db[docid] = doc except Exception as e: if not ignore_errors: raise print('Error: %s' % e, file=sys.stderr)
def _request_json(self, method, path=None, body=None, headers=None, **params): status, headers, data = self._request(method, path, body=body, headers=headers, **params) if 'application/json' in headers.get('content-type', ''): data = json.decode(data.read().decode('utf-8')) return status, headers, data
def _changes(self, **opts): opts.setdefault('yield_beats', False) yield_beats = opts.pop('yield_beats') _, _, data = self.resource.get('_changes', **opts) lines = iter(data) for ln in lines: if not ln: # skip heartbeats if yield_beats: yield {} continue doc = json.decode(ln) if 'last_seq' in doc: # consume the rest of the response if this for ln in lines: # was the last line, allows conn reuse pass yield doc
def get_json_raw_url(mb, path): """ little workaround for skirting overzealous couchdb-python quoting behavior. """ from couchdb import http, json session = mb.resource.session creds = mb.resource.credentials method = "GET" path = [mb.resource.url] + path url = http.urljoin(*path) status, headers, data = session.request(method, url, credentials=creds) if "application/json" in headers.get("content-type"): data = json.decode(data.read()) return status, headers, data
def run(self): url = self._path() res = Requester().get(url, headers={'accept': 'application/json'}) res_data = res.read() res.close() doc = couchdb_json.decode(res_data) doc_count = doc['doc_count'] with couchdb_write_multipart(sys.stdout, boundary=None) as envelope: done = 0 sys.stderr.write("Doc Count: %d\n" % doc_count) sys.stderr.write("Chunk Size: %d\n" % self._chunk_size) while done < doc_count: batch_size = self._run_chunk(envelope, done) if batch_size == 0: break else: done += batch_size sys.stderr.write("%.2f%% %d/%d\n" % \ ((float(done) / float(doc_count) * 100), done, doc_count)) sys.stderr.write("\n") pass
def copy(self, src, dest): """Copy the given document to create a new document. :param src: the ID of the document to copy, or a dictionary or `Document` object representing the source document. :param dest: either the destination document ID as string, or a dictionary or `Document` instance of the document that should be overwritten. :return: the new revision of the destination document :rtype: `str` :since: 0.6 """ if not isinstance(src, basestring): if not isinstance(src, dict): if hasattr(src, 'items'): src = dict(src.items()) else: raise TypeError('expected dict or string, got %s' % type(src)) src = src['_id'] if not isinstance(dest, basestring): if not isinstance(dest, dict): if hasattr(dest, 'items'): dest = dict(dest.items()) else: raise TypeError('expected dict or string, got %s' % type(dest)) if '_rev' in dest: dest = '%s?%s' % (http.quote( dest['_id']), http.urlencode({'rev': dest['_rev']})) else: dest = http.quote(dest['_id']) _, _, data = self.resource._request('COPY', src, headers={'Destination': dest}) data = json.decode(data.read()) return data['rev']
def __call__(self): """Reads notifications from stdin and triggers replication""" options = self.options wait_counter = time.time() while True: # non-blocking readline(), raises IOErrors fcntl.fcntl(sys.stdin, fcntl.F_SETFL, os.O_NONBLOCK) try: line = sys.stdin.readline() # poor man's validation. If we get garbage, we sys.exit if not line.endswith('}\n'): sys.exit(0) note = json.decode(line) log.debug('Received %r', note) # we don't care for deletes if note['type'] == 'delete' and not options.ignore_deletes: continue self.databases.append((note['type'], note['db'])) # if there are more docs that we want to batch, flush if len(self.databases) >= int(options.batch_threshold): self.sync_databases() continue except IOError: # if we waited longer that we want to wait, flush if (time.time() - wait_counter) > int(options.wait_threshold): self.sync_databases() wait_counter = time.time() time.sleep(float(options.wait_threshold))
def copy(self, src, dest): """Copy the given document to create a new document. :param src: the ID of the document to copy, or a dictionary or `Document` object representing the source document. :param dest: either the destination document ID as string, or a dictionary or `Document` instance of the document that should be overwritten. :return: the new revision of the destination document :rtype: `str` :since: 0.6 """ if not isinstance(src, util.strbase): if not isinstance(src, dict): if hasattr(src, 'items'): src = dict(src.items()) else: raise TypeError('expected dict or string, got %s' % type(src)) src = src['_id'] if not isinstance(dest, util.strbase): if not isinstance(dest, dict): if hasattr(dest, 'items'): dest = dict(dest.items()) else: raise TypeError('expected dict or string, got %s' % type(dest)) if '_rev' in dest: dest = '%s?%s' % (http.quote(dest['_id']), http.urlencode({'rev': dest['_rev']})) else: dest = http.quote(dest['_id']) _, _, data = self.resource._request('COPY', src, headers={'Destination': dest}) data = json.decode(data.read().decode('utf-8')) return data['rev']
def revsion_map(database, dbm): results = dbm.database.query(map_form_model_for_subject_questionnaires) revid_map = {} for row in results: revision_ids = get_revisions_dict(database, row) log_statement("Form Model _id : %s " % row['key']) form_model = get_form_model(dbm, row['value']) if is_max_len_equal_to(form_model, 20): rev_with_20 = form_model.revision for previous_rev in revision_ids: revision_doc = urllib2.urlopen(SERVER + "/" + database + "/" + row['key'] + "?rev=" + previous_rev).read() revision_doc = get_form_model(dbm, json.decode(revision_doc)) if is_max_len_equal_to(revision_doc, 12): revid_map[previous_rev] = (rev_with_20, form_model.form_code) log_statement( "added revision for form:%s old:%s new:%s " % (form_model.form_code, previous_rev, rev_with_20)) break rev_with_20 = previous_rev return revid_map
results = function(keys, vals) return [True, [results]] def rereduce(*cmd): return reduce(*cmd, **{'rereduce': True}) handlers = {'reset': reset, 'add_fun': add_fun, 'map_doc': map_doc, 'reduce': reduce, 'rereduce': rereduce} try: while True: line = input.readline() if not line: break try: cmd = json.decode(line) log.debug('Processing %r', cmd) except ValueError, e: log.error('Error: %s', e, exc_info=True) return 1 else: retval = handlers[cmd[0]](*cmd[1:]) log.debug('Returning %r', retval) _writejson(retval) except KeyboardInterrupt: return 0 except Exception, e: log.error('Error: %s', e, exc_info=True) return 1
def request(self, method, url, body=None, headers=None, credentials=None, num_redirects=0): if url in self.perm_redirects: url = self.perm_redirects[url] method = method.upper() if headers is None: headers = {} headers.setdefault('Accept', 'application/json') headers['User-Agent'] = self.user_agent cached_resp = None if method in ('GET', 'HEAD'): cached_resp = self.cache.get(url) if cached_resp is not None: etag = cached_resp[1].get('etag') if etag: headers['If-None-Match'] = etag if (body is not None and not isinstance(body, util.strbase) and not hasattr(body, 'read')): body = json.encode(body).encode('utf-8') headers.setdefault('Content-Type', 'application/json') if body is None: headers.setdefault('Content-Length', '0') elif isinstance(body, util.strbase): headers.setdefault('Content-Length', str(len(body))) else: headers['Transfer-Encoding'] = 'chunked' authorization = basic_auth(credentials) if authorization: headers['Authorization'] = authorization path_query = util.urlunsplit(('', '') + util.urlsplit(url)[2:4] + ('', )) conn = self.connection_pool.get(url) def _try_request_with_retries(retries): while True: try: return _try_request() except socket.error as e: ecode = e.args[0] if ecode not in self.retryable_errors: raise try: delay = next(retries) except StopIteration: # No more retries, raise last socket error. raise e finally: time.sleep(delay) conn.close() def _try_request(): try: conn.putrequest(method, path_query, skip_accept_encoding=True) for header in headers: conn.putheader(header, headers[header]) if body is None: conn.endheaders() else: if isinstance(body, util.strbase): if isinstance(body, util.utype): conn.endheaders(body.encode('utf-8')) else: conn.endheaders(body) else: # assume a file-like object and send in chunks conn.endheaders() while 1: chunk = body.read(CHUNK_SIZE) if not chunk: break if isinstance(chunk, util.utype): chunk = chunk.encode('utf-8') status = ('%x\r\n' % len(chunk)).encode('utf-8') conn.send(status + chunk + b'\r\n') conn.send(b'0\r\n\r\n') return conn.getresponse() except BadStatusLine as e: # httplib raises a BadStatusLine when it cannot read the status # line saying, "Presumably, the server closed the connection # before sending a valid response." # Raise as ECONNRESET to simplify retry logic. if e.line == '' or e.line == "''": raise socket.error(errno.ECONNRESET) else: raise resp = _try_request_with_retries(iter(self.retry_delays)) status = resp.status # Handle conditional response if status == 304 and method in ('GET', 'HEAD'): resp.read() self.connection_pool.release(url, conn) status, msg, data = cached_resp if data is not None: data = util.StringIO(data) return status, msg, data elif cached_resp: self.cache.remove(url) # Handle redirects if status == 303 or \ method in ('GET', 'HEAD') and status in (301, 302, 307): resp.read() self.connection_pool.release(url, conn) if num_redirects > self.max_redirects: raise RedirectLimit('Redirection limit exceeded') location = resp.getheader('location') if status == 301: self.perm_redirects[url] = location elif status == 303: method = 'GET' return self.request(method, location, body, headers, num_redirects=num_redirects + 1) data = None streamed = False # Read the full response for empty responses so that the connection is # in good state for the next request if method == 'HEAD' or resp.getheader('content-length') == '0' or \ status < 200 or status in (204, 304): resp.read() self.connection_pool.release(url, conn) # Buffer small non-JSON response bodies elif int(resp.getheader('content-length', sys.maxsize)) < CHUNK_SIZE: data = resp.read() self.connection_pool.release(url, conn) # For large or chunked response bodies, do not buffer the full body, # and instead return a minimal file-like object else: data = ResponseBody(resp, self.connection_pool, url, conn) streamed = True # Handle errors if status >= 400: ctype = resp.getheader('content-type') if data is not None and 'application/json' in ctype: data = json.decode(data.decode('utf-8')) error = data.get('error'), data.get('reason') elif method != 'HEAD': error = resp.read() self.connection_pool.release(url, conn) else: error = '' if status == 401: raise Unauthorized(error) elif status == 404: raise ResourceNotFound(error) elif status == 409: raise ResourceConflict(error) elif status == 412: raise PreconditionFailed(error) else: raise ServerError((status, error)) # Store cachable responses if not streamed and method == 'GET' and 'etag' in resp.msg: self.cache.put(url, (status, resp.msg, data)) if not streamed and data is not None: data = util.StringIO(data) return status, resp.msg, data
def run(input=sys.stdin, output=sys.stdout): r"""CouchDB view function handler implementation for Python. :param input: the readable file-like object to read input from :param output: the writable file-like object to write output to """ functions = [] def _writejson(obj): obj = json.encode(obj) if isinstance(obj, util.utype): obj = obj.encode('utf-8') output.write(obj) output.write(b'\n') output.flush() def _log(message): if not isinstance(message, util.strbase): message = json.encode(message) _writejson({'log': message}) def reset(config=None): del functions[:] return True def add_fun(string): string = BOM_UTF8 + string.encode('utf-8') globals_ = {} try: util.pyexec(string, {'log': _log}, globals_) except Exception as e: return { 'error': { 'id': 'map_compilation_error', 'reason': e.args[0] } } err = { 'error': { 'id': 'map_compilation_error', 'reason': 'string must eval to a function ' '(ex: "def(doc): return 1")' } } if len(globals_) != 1: return err function = list(globals_.values())[0] if type(function) is not FunctionType: return err functions.append(function) return True def map_doc(doc): results = [] for function in functions: try: results.append([[key, value] for key, value in function(doc)]) except Exception as e: log.error('runtime error in map function: %s', e, exc_info=True) results.append([]) _log(traceback.format_exc()) return results def reduce(*cmd, **kwargs): code = BOM_UTF8 + cmd[0][0].encode('utf-8') args = cmd[1] globals_ = {} try: util.pyexec(code, {'log': _log}, globals_) except Exception as e: log.error('runtime error in reduce function: %s', e, exc_info=True) return { 'error': { 'id': 'reduce_compilation_error', 'reason': e.args[0] } } err = { 'error': { 'id': 'reduce_compilation_error', 'reason': 'string must eval to a function ' '(ex: "def(keys, values): return 1")' } } if len(globals_) != 1: return err function = list(globals_.values())[0] if type(function) is not FunctionType: return err rereduce = kwargs.get('rereduce', False) results = [] if rereduce: keys = None vals = args else: if args: keys, vals = zip(*args) else: keys, vals = [], [] if util.funcode(function).co_argcount == 3: results = function(keys, vals, rereduce) else: results = function(keys, vals) return [True, [results]] def rereduce(*cmd): # Note: weird kwargs is for Python 2.5 compat return reduce(*cmd, **{'rereduce': True}) handlers = { 'reset': reset, 'add_fun': add_fun, 'map_doc': map_doc, 'reduce': reduce, 'rereduce': rereduce } try: while True: line = input.readline() if not line: break try: cmd = json.decode(line) log.debug('Processing %r', cmd) except ValueError as e: log.error('Error: %s', e, exc_info=True) return 1 else: retval = handlers[cmd[0]](*cmd[1:]) log.debug('Returning %r', retval) _writejson(retval) except KeyboardInterrupt: return 0 except Exception as e: log.error('Error: %s', e, exc_info=True) return 1
elif int(resp.getheader('content-length', sys.maxint)) < CHUNK_SIZE: data = resp.read() self.connection_pool.release(url, conn) # For large or chunked response bodies, do not buffer the full body, # and instead return a minimal file-like object else: data = ResponseBody(resp, lambda: self.connection_pool.release(url, conn)) streamed = True # Handle errors if status >= 400: ctype = resp.getheader('content-type') if data is not None and 'application/json' in ctype: data = json.decode(data) error = data.get('error'), data.get('reason') elif method != 'HEAD': error = resp.read() self.connection_pool.release(url, conn) else: error = '' if status == 401: raise Unauthorized(error) elif status == 404: raise ResourceNotFound(error) elif status == 409: raise ResourceConflict(error) elif status == 412: raise PreconditionFailed(error) else:
def run(input=sys.stdin, output=sys.stdout): r"""CouchDB view function handler implementation for Python. :param input: the readable file-like object to read input from :param output: the writable file-like object to write output to """ functions = [] def _writejson(obj): obj = json.encode(obj) if isinstance(obj, util.utype): obj = obj.encode('utf-8') output.write(obj) output.write(b'\n') output.flush() def _log(message): if not isinstance(message, util.strbase): message = json.encode(message) _writejson({'log': message}) def reset(config=None): del functions[:] return True def add_fun(string): string = BOM_UTF8 + string.encode('utf-8') globals_ = {} try: util.pyexec(string, {'log': _log}, globals_) except Exception as e: return {'error': { 'id': 'map_compilation_error', 'reason': e.args[0] }} err = {'error': { 'id': 'map_compilation_error', 'reason': 'string must eval to a function ' '(ex: "def(doc): return 1")' }} if len(globals_) != 1: return err function = list(globals_.values())[0] if type(function) is not FunctionType: return err functions.append(function) return True def map_doc(doc): results = [] for function in functions: try: results.append([[key, value] for key, value in function(doc)]) except Exception as e: log.error('runtime error in map function: %s', e, exc_info=True) results.append([]) _log(traceback.format_exc()) return results def reduce(*cmd, **kwargs): code = BOM_UTF8 + cmd[0][0].encode('utf-8') args = cmd[1] globals_ = {} try: util.pyexec(code, {'log': _log}, globals_) except Exception as e: log.error('runtime error in reduce function: %s', e, exc_info=True) return {'error': { 'id': 'reduce_compilation_error', 'reason': e.args[0] }} err = {'error': { 'id': 'reduce_compilation_error', 'reason': 'string must eval to a function ' '(ex: "def(keys, values): return 1")' }} if len(globals_) != 1: return err function = list(globals_.values())[0] if type(function) is not FunctionType: return err rereduce = kwargs.get('rereduce', False) results = [] if rereduce: keys = None vals = args else: if args: keys, vals = zip(*args) else: keys, vals = [], [] if util.funcode(function).co_argcount == 3: results = function(keys, vals, rereduce) else: results = function(keys, vals) return [True, [results]] def rereduce(*cmd): # Note: weird kwargs is for Python 2.5 compat return reduce(*cmd, **{'rereduce': True}) handlers = {'reset': reset, 'add_fun': add_fun, 'map_doc': map_doc, 'reduce': reduce, 'rereduce': rereduce} try: while True: line = input.readline() if not line: break try: cmd = json.decode(line) log.debug('Processing %r', cmd) except ValueError as e: log.error('Error: %s', e, exc_info=True) return 1 else: retval = handlers[cmd[0]](*cmd[1:]) log.debug('Returning %r', retval) _writejson(retval) except KeyboardInterrupt: return 0 except Exception as e: log.error('Error: %s', e, exc_info=True) return 1
def put_json(self, *a, **k): status, headers, data = self.put(*a, **k) if 'application/json' in headers.get('content-type'): data = json.decode(data.read()) return status, headers, data
elif int(resp.getheader('content-length', sys.maxint)) < CHUNK_SIZE: data = resp.read() self._return_connection(url, conn) # For large or chunked response bodies, do not buffer the full body, # and instead return a minimal file-like object else: data = ResponseBody(resp, lambda: self._return_connection(url, conn)) streamed = True # Handle errors if status >= 400: ctype = resp.getheader('content-type') if data is not None and 'application/json' in ctype: data = json.decode(data) error = data.get('error'), data.get('reason') elif method != 'HEAD': error = resp.read() self._return_connection(url, conn) else: error = '' if status == 401: raise Unauthorized(error) elif status == 404: raise ResourceNotFound(error) elif status == 409: raise ResourceConflict(error) elif status == 412: raise PreconditionFailed(error) else:
def request(self, method, url, body=None, headers=None, credentials=None, num_redirects=0): if url in self.perm_redirects: url = self.perm_redirects[url] method = method.upper() if headers is None: headers = {} headers.setdefault('Accept', 'application/json') headers['User-Agent'] = self.user_agent cached_resp = None if method in ('GET', 'HEAD'): cached_resp = self.cache.get(url) if cached_resp is not None: etag = cached_resp[1].get('etag') if etag: headers['If-None-Match'] = etag if (body is not None and not isinstance(body, util.strbase) and not hasattr(body, 'read')): body = json.encode(body).encode('utf-8') headers.setdefault('Content-Type', 'application/json') if body is None: headers.setdefault('Content-Length', '0') elif isinstance(body, util.strbase): headers.setdefault('Content-Length', str(len(body))) else: headers['Transfer-Encoding'] = 'chunked' authorization = basic_auth(credentials) if authorization: headers['Authorization'] = authorization path_query = util.urlunsplit(('', '') + util.urlsplit(url)[2:4] + ('',)) conn = self.connection_pool.get(url) def _try_request_with_retries(retries): while True: try: return _try_request() except socket.error as e: ecode = e.args[0] if ecode not in self.retryable_errors: raise try: delay = retries.next() except StopIteration: # No more retries, raise last socket error. raise e time.sleep(delay) conn.close() def _try_request(): try: conn.putrequest(method, path_query, skip_accept_encoding=True) for header in headers: conn.putheader(header, headers[header]) if body is None: conn.endheaders() else: if isinstance(body, util.strbase): if isinstance(body, util.utype): conn.endheaders(body.encode('utf-8')) else: conn.endheaders(body) else: # assume a file-like object and send in chunks conn.endheaders() while 1: chunk = body.read(CHUNK_SIZE) if not chunk: break if isinstance(chunk, util.utype): chunk = chunk.encode('utf-8') status = ('%x\r\n' % len(chunk)).encode('utf-8') conn.send(status + chunk + b'\r\n') conn.send(b'0\r\n\r\n') return conn.getresponse() except BadStatusLine as e: # httplib raises a BadStatusLine when it cannot read the status # line saying, "Presumably, the server closed the connection # before sending a valid response." # Raise as ECONNRESET to simplify retry logic. if e.line == '' or e.line == "''": raise socket.error(errno.ECONNRESET) else: raise resp = _try_request_with_retries(iter(self.retry_delays)) status = resp.status # Handle conditional response if status == 304 and method in ('GET', 'HEAD'): resp.read() self.connection_pool.release(url, conn) status, msg, data = cached_resp if data is not None: data = util.StringIO(data) return status, msg, data elif cached_resp: self.cache.remove(url) # Handle redirects if status == 303 or \ method in ('GET', 'HEAD') and status in (301, 302, 307): resp.read() self.connection_pool.release(url, conn) if num_redirects > self.max_redirects: raise RedirectLimit('Redirection limit exceeded') location = resp.getheader('location') if status == 301: self.perm_redirects[url] = location elif status == 303: method = 'GET' return self.request(method, location, body, headers, num_redirects=num_redirects + 1) data = None streamed = False # Read the full response for empty responses so that the connection is # in good state for the next request if method == 'HEAD' or resp.getheader('content-length') == '0' or \ status < 200 or status in (204, 304): resp.read() self.connection_pool.release(url, conn) # Buffer small non-JSON response bodies elif int(resp.getheader('content-length', sys.maxsize)) < CHUNK_SIZE: data = resp.read() self.connection_pool.release(url, conn) # For large or chunked response bodies, do not buffer the full body, # and instead return a minimal file-like object else: data = ResponseBody(resp, self.connection_pool, url, conn) streamed = True # Handle errors if status >= 400: ctype = resp.getheader('content-type') if data is not None and 'application/json' in ctype: data = json.decode(data.decode('utf-8')) error = data.get('error'), data.get('reason') elif method != 'HEAD': error = resp.read() self.connection_pool.release(url, conn) else: error = '' if status == 401: raise Unauthorized(error) elif status == 404: raise ResourceNotFound(error) elif status == 409: raise ResourceConflict(error) elif status == 412: raise PreconditionFailed(error) else: raise ServerError((status, error)) # Store cachable responses if not streamed and method == 'GET' and 'etag' in resp.msg: self.cache.put(url, (status, resp.msg, data)) if not streamed and data is not None: data = util.StringIO(data) return status, resp.msg, data
def _request_json(self, method, path=None, body=None, headers=None, **params): status, headers, data = self._request(method, path, body=body, headers=headers, **params) if "application/json" in headers.get("content-type"): data = json.decode(data.read()) return status, headers, data
return [True, [results]] def rereduce(*cmd): # Note: weird kwargs is for Python 2.5 compat return reduce(*cmd, **{'rereduce': True}) handlers = {'reset': reset, 'add_fun': add_fun, 'map_doc': map_doc, 'reduce': reduce, 'rereduce': rereduce} try: while True: line = input.readline() if not line: break try: cmd = json.decode(line) log.debug('Processing %r', cmd) except ValueError, e: log.error('Error: %s', e, exc_info=True) return 1 else: retval = handlers[cmd[0]](*cmd[1:]) log.debug('Returning %r', retval) _writejson(retval) except KeyboardInterrupt: return 0 except Exception, e: log.error('Error: %s', e, exc_info=True) return 1
class Resource(object): def __init__(self, http, uri): if http is None: http = httplib2.Http() http.force_exception_to_status_code = False self.http = http self.uri = uri def __call__(self, path): return type(self)(self.http, uri(self.uri, path)) def delete(self, path=None, headers=None, **params): return self._request('DELETE', path, headers=headers, **params) def get(self, path=None, headers=None, **params): return self._request('GET', path, headers=headers, **params) def head(self, path=None, headers=None, **params): return self._request('HEAD', path, headers=headers, **params) def post(self, path=None, content=None, headers=None, **params): return self._request('POST', path, content=content, headers=headers, **params) def put(self, path=None, content=None, headers=None, **params): return self._request('PUT', path, content=content, headers=headers, **params) def _request(self, method, path=None, content=None, headers=None, **params): from couchdb import __version__ headers = headers or {} headers.setdefault('Accept', 'application/json') headers.setdefault('User-Agent', 'couchdb-python %s' % __version__) body = None if content is not None: if not isinstance(content, basestring): body = json.encode(content).encode('utf-8') headers.setdefault('Content-Type', 'application/json') else: body = content headers.setdefault('Content-Length', str(len(body))) def _make_request(retry=1): try: return self.http.request(uri(self.uri, path, **params), method, body=body, headers=headers) except socket.error, e: if retry > 0 and e.args[0] == 54: # reset by peer return _make_request(retry - 1) raise resp, data = _make_request() status_code = int(resp.status) if data and resp.get('content-type') == 'application/json': try: data = json.decode(data) except ValueError: pass if status_code >= 400: if type(data) is dict: error = (data.get('error'), data.get('reason')) else: error = data if status_code == 404: raise ResourceNotFound(error) elif status_code == 409: raise ResourceConflict(error) elif status_code == 412: raise PreconditionFailed(error) else: raise ServerError((status_code, error)) return resp, data