def dump_docs(envelope, db, docs): for doc in docs: print >> sys.stderr, 'Dumping document %r' % doc.id attachments = doc.pop('_attachments', {}) jsondoc = json.encode(doc) if attachments: parts = envelope.open({ 'Content-ID': doc.id, 'ETag': '"%s"' % doc.rev }) parts.add('application/json', jsondoc) for name, info in attachments.items(): content_type = info.get('content_type') if content_type is None: # CouchDB < 0.8 content_type = info.get('content-type') if 'data' not in info: data = db.get_attachment(doc, name).read() else: data = b64decode(info['data']) parts.add(content_type, data, {'Content-ID': name}) parts.close() else: envelope.add('application/json', jsondoc, { 'Content-ID': doc.id, 'ETag': '"%s"' % doc.rev })
def dump_docs(envelope, _ids, db): for _id in _ids: doc = db.get(_id, attachments=True) print >> sys.stderr, 'Dumping document %r' % doc.id attachments = doc.pop('_attachments', {}) jsondoc = json.encode(doc) if attachments: parts = envelope.open({ 'Content-ID': doc.id, 'ETag': '"%s"' % doc.rev }) parts.add('application/json', jsondoc) for name, info in attachments.items(): content_type = info.get('content_type') if content_type is None: # CouchDB < 0.8 content_type = info.get('content-type') parts.add(content_type, b64decode(info['data']), { 'Content-ID': name }) parts.close() else: envelope.add('application/json', jsondoc, { 'Content-ID': doc.id, 'ETag': '"%s"' % doc.rev })
def dump_docs(envelope, docs): for doc in docs: print('Dumping document %r' % doc.id, file=sys.stderr) attachments = doc.pop('_attachments', {}) jsondoc = json.encode(doc) if attachments: parts = envelope.open({ 'Content-ID': doc.id, 'ETag': '"%s"' % doc.rev }) parts.add('application/json', jsondoc) for name, info in attachments.items(): content_type = info.get('content_type') if content_type is None: # CouchDB < 0.8 content_type = info.get('content-type') parts.add(content_type, b64decode(info['data']), {'Content-ID': name}) parts.close() else: envelope.add('application/json', jsondoc, { 'Content-ID': doc.id, 'ETag': '"%s"' % doc.rev })
def _writejson(obj): obj = json.encode(obj) if isinstance(obj, str): obj = obj.encode('utf-8') output.write(obj) output.write('\n') output.flush()
def _writejson(obj): obj = json.encode(obj) if isinstance(obj, util.utype): obj = obj.encode('utf-8') output.write(obj) output.write(b'\n') output.flush()
def _process_row(self, envelope, doc): """ Processes a row See: couchdb.tools.dump """ attachments = doc.pop('_attachments', {}) jsondoc = couchdb_json.encode(doc) if attachments: parts = envelope.open({ 'Content-ID': doc['_id'], 'ETag': '"%s"' % doc['_rev'] }) parts.add('application/json', jsondoc) for name, info in attachments.items(): content_type = info.get('content_type') ## CouchDB < 0.8 if content_type is None: content_type = info.get('content-type') parts.add(content_type, base64.b64decode(info['data']), { 'Content-ID': name }) parts.close() else: envelope.add('application/json;charset=utf-8', jsondoc, { 'Content-ID': doc['_id'], 'ETag': '"%s"' % doc['_rev'] }) return True
def request(self, method, url, body=None, headers=None, credentials=None, num_redirects=0): if url in self.perm_redirects: url = self.perm_redirects[url] method = method.upper() if headers is None: headers = {} headers.setdefault('Accept', 'application/json') headers['User-Agent'] = self.user_agent cached_resp = None if method in ('GET', 'HEAD'): cached_resp = self.cache.get(url) if cached_resp is not None: etag = cached_resp[1].get('etag') if etag: headers['If-None-Match'] = etag if body is None: headers.setdefault('Content-Length', '0') else: if not isinstance(body, basestring): try: body = json.encode(body).encode('utf-8') except TypeError: # Check for somethine file-like or re-raise the exception # to avoid masking real JSON encoding errors. if not hasattr(body, 'read'): raise else: headers.setdefault('Content-Type', 'application/json') if isinstance(body, basestring): headers.setdefault('Content-Length', str(len(body))) else: headers['Transfer-Encoding'] = 'chunked' authorization = basic_auth(credentials) if authorization: headers['Authorization'] = authorization path_query = urlunsplit(('', '') + urlsplit(url)[2:4] + ('',)) conn = self._get_connection(url) def _try_request_with_retries(retries): while True: try: return _try_request() except socket.error, e: ecode = e.args[0] if ecode not in self.retryable_errors: raise try: delay = retries.next() except StopIteration: # No more retries, raise last socket error. raise e time.sleep(delay) conn.close()
def _encode_options(self, options): retval = {} for name, value in options.items(): if name in ('key', 'startkey', 'endkey') \ or not isinstance(value, basestring): value = json.encode(value) retval[name] = value return retval
def request(self, method, url, body=None, headers=None, credentials=None, num_redirects=0): if url in self.perm_redirects: url = self.perm_redirects[url] method = method.upper() if headers is None: headers = {} headers.setdefault('Accept', 'application/json') headers['User-Agent'] = self.user_agent cached_resp = None if method in ('GET', 'HEAD'): cached_resp = self.cache.get(url) if cached_resp is not None: etag = cached_resp[1].get('etag') if etag: headers['If-None-Match'] = etag if (body is not None and not isinstance(body, basestring) and not hasattr(body, 'read')): body = json.encode(body).encode('utf-8') headers.setdefault('Content-Type', 'application/json') if body is None: headers.setdefault('Content-Length', '0') elif isinstance(body, basestring): headers.setdefault('Content-Length', str(len(body))) else: headers['Transfer-Encoding'] = 'chunked' authorization = basic_auth(credentials) if authorization: headers['Authorization'] = authorization path_query = urlunsplit(('', '') + urlsplit(url)[2:4] + ('', )) conn = self.connection_pool.get(url) def _try_request_with_retries(retries): while True: try: return _try_request() except socket.error, e: ecode = e.args[0] if ecode not in self.retryable_errors: raise try: delay = retries.next() except StopIteration: # No more retries, raise last socket error. raise e time.sleep(delay) conn.close()
def tmp(docIdx): resp, content = http.request('%s/%s/%s' % (params.server, params.testdb, docs[i]), 'GET') obj = json.decode(content) if obj.has_key('counter'): obj['counter'] = obj['counter'] + 1 else: obj['counter'] = 0 doc = json.encode(obj) http.request('%s/%s/%s' % (params.server, params.testdb, docs[i]), 'PUT', body=doc)
def _encode_view_options(options): """Encode any items in the options dict that are sent as a JSON string to a view/list function. """ retval = {} for name, value in options.items(): if name in ("key", "startkey", "endkey") or not isinstance(value, basestring): value = json.encode(value) retval[name] = value return retval
def _encode_view_options(options): """Encode any items in the options dict that are sent as a JSON string to a view/list function. """ retval = {} for name, value in options.items(): if name in ('key', 'startkey', 'endkey') \ or not isinstance(value, util.strbase): value = json.encode(value) retval[name] = value return retval
def _exec(self, options): body = {"map": self.map_fun, "language": self.language} if self.reduce_fun: body["reduce"] = self.reduce_fun if "keys" in options: options = options.copy() body["keys"] = options.pop("keys") content = json.encode(body).encode("utf-8") _, _, data = self.resource.post_json( body=content, headers={"Content-Type": "application/json"}, **_encode_view_options(options) ) return data
def _exec(self, options): body = {'map': self.map_fun, 'language': self.language} if self.reduce_fun: body['reduce'] = self.reduce_fun if 'keys' in options: options = options.copy() body['keys'] = options.pop('keys') content = json.encode(body).encode('utf-8') _, _, data = self.resource.post_json(body=content, headers={ 'Content-Type': 'application/json' }, **_encode_view_options(options)) return data
def request(self, method, url, body=None, headers=None, credentials=None, num_redirects=0): if url in self.perm_redirects: url = self.perm_redirects[url] method = method.upper() if headers is None: headers = {} headers.setdefault("Accept", "application/json") headers["User-Agent"] = self.user_agent cached_resp = None if method in ("GET", "HEAD"): cached_resp = self.cache.get(url) if cached_resp is not None: etag = cached_resp[1].get("etag") if etag: headers["If-None-Match"] = etag if body is not None and not isinstance(body, basestring) and not hasattr(body, "read"): body = json.encode(body).encode("utf-8") headers.setdefault("Content-Type", "application/json") if body is None: headers.setdefault("Content-Length", "0") elif isinstance(body, basestring): headers.setdefault("Content-Length", str(len(body))) else: headers["Transfer-Encoding"] = "chunked" authorization = basic_auth(credentials) if authorization: headers["Authorization"] = authorization path_query = urlunsplit(("", "") + urlsplit(url)[2:4] + ("",)) conn = self.connection_pool.get(url) def _try_request_with_retries(retries): while True: try: return _try_request() except socket.error, e: ecode = e.args[0] if ecode not in self.retryable_errors: raise try: delay = retries.next() except StopIteration: # No more retries, raise last socket error. raise e time.sleep(delay) conn.close()
def test_posts(self): print "\ntesting POST...\n" for url in self.bus: data = urllib.urlopen(url).read() data = json.loads(data.strip()) for k in data.keys(): if k == 'description': data['description'] = 'todays business was created %s' % datetime.today() elif k != '_id' and k != '_rev': del data[k] data = j.encode(data).encode('utf-8') print "POST data = %s" % data headers = { "content-type": "application/x-www-form-urlencoded", 'content-length': str(len(data))} response, content = httplib2.Http().request(url, "POST", body = data, headers = headers) print "\t[%s] POST %s" % (response['status'], url) self.assertEqual(response['status'], '201')
def dump_docs(envelope, db, docs): for doc in docs: print('Dumping document %r' % doc.id, file=sys.stderr) attachments = doc.pop('_attachments', {}) jsondoc = json.encode(doc) if attachments: open_envelope = False for name, info in attachments.items(): content_type = info.get('content_type') if content_type is None: # CouchDB < 0.8 content_type = info.get('content-type') if 'data' not in info: attachment = db.get_attachment(doc, name) # In case the attachment is not in the db if attachment is None: data = {} print('Missing attachment') else: data = attachment.read() else: try: data = b64decode(info['data']) except TypeError as e: print('Could not decode attachment') data = {} if data and content_type: if not open_envelope: parts = envelope.open({ 'Content-ID': doc.id, 'ETag': '"%s"' % doc.rev }) parts.add('application/json', jsondoc) open_envelope = True parts.add(content_type, data, {'Content-ID': name}) parts.close() else: envelope.add('application/json', jsondoc, { 'Content-ID': doc.id, 'ETag': '"%s"' % doc.rev })
def _request(self, method, path=None, content=None, headers=None, **params): from couchdb import __version__ headers = headers or {} headers.setdefault('Accept', 'application/json') headers.setdefault('User-Agent', 'couchdb-python %s' % __version__) body = None if content is not None: if not isinstance(content, basestring): body = json.encode(content).encode('utf-8') headers.setdefault('Content-Type', 'application/json') else: body = content headers.setdefault('Content-Length', str(len(body))) def _make_request(retry=1): try: return self.http.request(uri(self.uri, path, **params), method, body=body, headers=headers) except socket.error, e: if retry > 0 and e.args[0] == 54: # reset by peer return _make_request(retry - 1) raise
def trigger_replication(self, database): """Triggers replication between source and target servers.""" log.debug('Replicate database %r', database) body = {'source': self.concat_uri(self.source, database)} # send replication request to target server for target in self.targets: body['target'] = database if target['username'] is not None: self.http.add_credentials(target['username'], target['password']) log.debug('Request replication %r from %s', body, target['scheme'] + target['host']) resp, data = self.http.request( self.concat_uri(target['scheme'] + target['host'], '_replicate'), 'POST', body=json.encode(body)) if resp.status != 200: log.error('Unexpected HTTP response: %s %s (%s)', resp.status, resp.reason, data) self.http.clear_credentials()
def dump_db(dburl, username=None, password=None, boundary=None, output=sys.stdout): db = Database(dburl) if username is not None and password is not None: db.resource.credentials = username, password envelope = write_multipart(output, boundary=boundary) for docid in db: doc = db.get(docid, attachments=True) print >> sys.stderr, 'Dumping document %r' % doc.id attachments = doc.pop('_attachments', {}) jsondoc = json.encode(doc) if attachments: parts = envelope.open({ 'Content-ID': doc.id, 'ETag': '"%s"' % doc.rev }) parts.add('application/json', jsondoc) for name, info in attachments.items(): content_type = info.get('content_type') if content_type is None: # CouchDB < 0.8 content_type = info.get('content-type') parts.add(content_type, b64decode(info['data']), { 'Content-ID': name }) parts.close() else: envelope.add('application/json', jsondoc, { 'Content-ID': doc.id, 'ETag': '"%s"' % doc.rev }) envelope.close()
def dump_db(dburl, username=None, password=None, boundary=None, output=sys.stdout): db = Database(dburl) if username is not None and password is not None: db.resource.credentials = username, password envelope = write_multipart(output, boundary=boundary) for docid in db: doc = db.get(docid, attachments=True) print('Dumping document %r' % doc.id, file=sys.stderr) attachments = doc.pop('_attachments', {}) jsondoc = json.encode(doc) if attachments: parts = envelope.open({ 'Content-ID': doc.id, 'ETag': '"%s"' % doc.rev }) parts.add('application/json', jsondoc) for name, info in list(attachments.items()): content_type = info.get('content_type') if content_type is None: # CouchDB < 0.8 content_type = info.get('content-type') parts.add(content_type, b64decode(info['data']), { 'Content-ID': name }) parts.close() else: envelope.add('application/json', jsondoc, { 'Content-ID': doc.id, 'ETag': '"%s"' % doc.rev }) envelope.close()
def request(self, method, url, body=None, headers=None, credentials=None, num_redirects=0): if url in self.perm_redirects: url = self.perm_redirects[url] method = method.upper() if headers is None: headers = {} headers.setdefault('Accept', 'application/json') headers['User-Agent'] = self.user_agent cached_resp = None if method in ('GET', 'HEAD'): cached_resp = self.cache.get(url) if cached_resp is not None: etag = cached_resp[1].get('etag') if etag: headers['If-None-Match'] = etag if (body is not None and not isinstance(body, util.strbase) and not hasattr(body, 'read')): body = json.encode(body).encode('utf-8') headers.setdefault('Content-Type', 'application/json') if body is None: headers.setdefault('Content-Length', '0') elif isinstance(body, util.strbase): headers.setdefault('Content-Length', str(len(body))) else: headers['Transfer-Encoding'] = 'chunked' authorization = basic_auth(credentials) if authorization: headers['Authorization'] = authorization path_query = util.urlunsplit(('', '') + util.urlsplit(url)[2:4] + ('', )) conn = self.connection_pool.get(url) def _try_request_with_retries(retries): while True: try: return _try_request() except socket.error as e: ecode = e.args[0] if ecode not in self.retryable_errors: raise try: delay = next(retries) except StopIteration: # No more retries, raise last socket error. raise e finally: time.sleep(delay) conn.close() def _try_request(): try: conn.putrequest(method, path_query, skip_accept_encoding=True) for header in headers: conn.putheader(header, headers[header]) if body is None: conn.endheaders() else: if isinstance(body, util.strbase): if isinstance(body, util.utype): conn.endheaders(body.encode('utf-8')) else: conn.endheaders(body) else: # assume a file-like object and send in chunks conn.endheaders() while 1: chunk = body.read(CHUNK_SIZE) if not chunk: break if isinstance(chunk, util.utype): chunk = chunk.encode('utf-8') status = ('%x\r\n' % len(chunk)).encode('utf-8') conn.send(status + chunk + b'\r\n') conn.send(b'0\r\n\r\n') return conn.getresponse() except BadStatusLine as e: # httplib raises a BadStatusLine when it cannot read the status # line saying, "Presumably, the server closed the connection # before sending a valid response." # Raise as ECONNRESET to simplify retry logic. if e.line == '' or e.line == "''": raise socket.error(errno.ECONNRESET) else: raise resp = _try_request_with_retries(iter(self.retry_delays)) status = resp.status # Handle conditional response if status == 304 and method in ('GET', 'HEAD'): resp.read() self.connection_pool.release(url, conn) status, msg, data = cached_resp if data is not None: data = util.StringIO(data) return status, msg, data elif cached_resp: self.cache.remove(url) # Handle redirects if status == 303 or \ method in ('GET', 'HEAD') and status in (301, 302, 307): resp.read() self.connection_pool.release(url, conn) if num_redirects > self.max_redirects: raise RedirectLimit('Redirection limit exceeded') location = resp.getheader('location') if status == 301: self.perm_redirects[url] = location elif status == 303: method = 'GET' return self.request(method, location, body, headers, num_redirects=num_redirects + 1) data = None streamed = False # Read the full response for empty responses so that the connection is # in good state for the next request if method == 'HEAD' or resp.getheader('content-length') == '0' or \ status < 200 or status in (204, 304): resp.read() self.connection_pool.release(url, conn) # Buffer small non-JSON response bodies elif int(resp.getheader('content-length', sys.maxsize)) < CHUNK_SIZE: data = resp.read() self.connection_pool.release(url, conn) # For large or chunked response bodies, do not buffer the full body, # and instead return a minimal file-like object else: data = ResponseBody(resp, self.connection_pool, url, conn) streamed = True # Handle errors if status >= 400: ctype = resp.getheader('content-type') if data is not None and 'application/json' in ctype: data = json.decode(data.decode('utf-8')) error = data.get('error'), data.get('reason') elif method != 'HEAD': error = resp.read() self.connection_pool.release(url, conn) else: error = '' if status == 401: raise Unauthorized(error) elif status == 404: raise ResourceNotFound(error) elif status == 409: raise ResourceConflict(error) elif status == 412: raise PreconditionFailed(error) else: raise ServerError((status, error)) # Store cachable responses if not streamed and method == 'GET' and 'etag' in resp.msg: self.cache.put(url, (status, resp.msg, data)) if not streamed and data is not None: data = util.StringIO(data) return status, resp.msg, data
def _log(message): if not isinstance(message, util.strbase): message = json.encode(message) _writejson({'log': message})
def put_json_always( resource, path, body =None): body = _json.encode( body).encode('utf-8') return resource.put_json( path, body= body)
def _log(message): if not isinstance(message, basestring): message = json.encode(message) _writejson({'log': message})
try: while True: line = input.readline() if not line: break try: cmd = json.decode(line) log.debug('Processing %r', cmd) except ValueError, e: log.error('Error: %s', e, exc_info=True) return 1 else: retval = handlers[cmd[0]](*cmd[1:]) log.debug('Returning %r', retval) output.write(json.encode(retval)) output.write('\n') output.flush() except KeyboardInterrupt: return 0 except Exception, e: log.error('Error: %s', e, exc_info=True) return 1 _VERSION = """%(name)s - CouchDB Python %(version)s Copyright (C) 2007 Christopher Lenz <*****@*****.**>. """ _HELP = """Usage: %(name)s [OPTION]
def _log(message): if not isinstance(message, basestring): message = json.encode(message) output.write(json.encode({'log': message})) output.write('\n') output.flush()
def _writejson(obj): obj = json.encode(obj) output.write(obj) output.write('\n') output.flush()
def request(self, method, url, body=None, headers=None, credentials=None, num_redirects=0): if url in self.perm_redirects: url = self.perm_redirects[url] method = method.upper() if headers is None: headers = {} headers.setdefault('Accept', 'application/json') headers['User-Agent'] = self.user_agent cached_resp = None if method in ('GET', 'HEAD'): cached_resp = self.cache.get(url) if cached_resp is not None: etag = cached_resp[1].get('etag') if etag: headers['If-None-Match'] = etag if (body is not None and not isinstance(body, util.strbase) and not hasattr(body, 'read')): body = json.encode(body).encode('utf-8') headers.setdefault('Content-Type', 'application/json') if body is None: headers.setdefault('Content-Length', '0') elif isinstance(body, util.strbase): headers.setdefault('Content-Length', str(len(body))) else: headers['Transfer-Encoding'] = 'chunked' authorization = basic_auth(credentials) if authorization: headers['Authorization'] = authorization path_query = util.urlunsplit(('', '') + util.urlsplit(url)[2:4] + ('',)) conn = self.connection_pool.get(url) def _try_request_with_retries(retries): while True: try: return _try_request() except socket.error as e: ecode = e.args[0] if ecode not in self.retryable_errors: raise try: delay = retries.next() except StopIteration: # No more retries, raise last socket error. raise e time.sleep(delay) conn.close() def _try_request(): try: conn.putrequest(method, path_query, skip_accept_encoding=True) for header in headers: conn.putheader(header, headers[header]) if body is None: conn.endheaders() else: if isinstance(body, util.strbase): if isinstance(body, util.utype): conn.endheaders(body.encode('utf-8')) else: conn.endheaders(body) else: # assume a file-like object and send in chunks conn.endheaders() while 1: chunk = body.read(CHUNK_SIZE) if not chunk: break if isinstance(chunk, util.utype): chunk = chunk.encode('utf-8') status = ('%x\r\n' % len(chunk)).encode('utf-8') conn.send(status + chunk + b'\r\n') conn.send(b'0\r\n\r\n') return conn.getresponse() except BadStatusLine as e: # httplib raises a BadStatusLine when it cannot read the status # line saying, "Presumably, the server closed the connection # before sending a valid response." # Raise as ECONNRESET to simplify retry logic. if e.line == '' or e.line == "''": raise socket.error(errno.ECONNRESET) else: raise resp = _try_request_with_retries(iter(self.retry_delays)) status = resp.status # Handle conditional response if status == 304 and method in ('GET', 'HEAD'): resp.read() self.connection_pool.release(url, conn) status, msg, data = cached_resp if data is not None: data = util.StringIO(data) return status, msg, data elif cached_resp: self.cache.remove(url) # Handle redirects if status == 303 or \ method in ('GET', 'HEAD') and status in (301, 302, 307): resp.read() self.connection_pool.release(url, conn) if num_redirects > self.max_redirects: raise RedirectLimit('Redirection limit exceeded') location = resp.getheader('location') if status == 301: self.perm_redirects[url] = location elif status == 303: method = 'GET' return self.request(method, location, body, headers, num_redirects=num_redirects + 1) data = None streamed = False # Read the full response for empty responses so that the connection is # in good state for the next request if method == 'HEAD' or resp.getheader('content-length') == '0' or \ status < 200 or status in (204, 304): resp.read() self.connection_pool.release(url, conn) # Buffer small non-JSON response bodies elif int(resp.getheader('content-length', sys.maxsize)) < CHUNK_SIZE: data = resp.read() self.connection_pool.release(url, conn) # For large or chunked response bodies, do not buffer the full body, # and instead return a minimal file-like object else: data = ResponseBody(resp, self.connection_pool, url, conn) streamed = True # Handle errors if status >= 400: ctype = resp.getheader('content-type') if data is not None and 'application/json' in ctype: data = json.decode(data.decode('utf-8')) error = data.get('error'), data.get('reason') elif method != 'HEAD': error = resp.read() self.connection_pool.release(url, conn) else: error = '' if status == 401: raise Unauthorized(error) elif status == 404: raise ResourceNotFound(error) elif status == 409: raise ResourceConflict(error) elif status == 412: raise PreconditionFailed(error) else: raise ServerError((status, error)) # Store cachable responses if not streamed and method == 'GET' and 'etag' in resp.msg: self.cache.put(url, (status, resp.msg, data)) if not streamed and data is not None: data = util.StringIO(data) return status, resp.msg, data
def get(self): if self.getCurrentUserType() == 'teacher' or self.getCurrentUserType() == 'admin': rawGenre = self.get_argument('genre').lower() rawStudentName = self.get_argument('studentName') rawSchoolName = self.get_argument('schoolName') nbTries = results.getNbTries(rawSchoolName, rawStudentName, rawGenre) lastText = results.getLastText(rawSchoolName, rawStudentName, rawGenre) lastCriterias = results.getLastCriterias(rawSchoolName, rawStudentName, rawGenre) lastComment = results.getLastComment(rawSchoolName, rawStudentName, rawGenre) result = '{ "result": { "genre": "'+rawGenre+'", "nbTries": "' + str(nbTries) + '","text": "' + lastText+ '","comment": "' + lastComment+ '", "criterias": ' +json.encode(lastCriterias) +' } }' self.set_header("Content-Type", "application/json") self.write(result) else: self.write("")