def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False):
db = Database(dburl)
if username is not None and password is not None:
db.resource.http.add_credentials(username, password)
for headers, is_multipart, payload in read_multipart(fileobj):
docid = headers['content-id']
if is_multipart: # doc has attachments
for headers, _, payload in payload:
if 'content-id' not in headers:
doc = json.decode(payload)
doc['_attachments'] = {}
else:
doc['_attachments'][headers['content-id']] = {
'data': b64encode(payload),
'content_type': headers['content-type'],
'length': len(payload)
}
else: # no attachments, just the JSON
doc = json.decode(payload)
del doc['_rev']
sys.stderr.write('Loading document %r' % docid)
try:
db[docid] = doc
except Exception as e:
if not ignore_errors:
raise
sys.stderr.write('Error: %s' % e)
def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False):
db = Database(dburl)
if username is not None and password is not None:
db.resource.credentials = (username, password)
for headers, is_multipart, payload in read_multipart(fileobj):
docid = headers['content-id']
if is_multipart: # doc has attachments
for headers, _, payload in payload:
if 'content-id' not in headers:
doc = json.decode(payload)
doc['_attachments'] = {}
else:
doc['_attachments'][headers['content-id']] = {
'data': b64encode(payload),
'content_type': headers['content-type'],
'length': len(payload)
}
else: # no attachments, just the JSON
doc = json.decode(payload)
del doc['_rev']
print >> sys.stderr, 'Loading document %r' % docid
try:
db[docid] = doc
except Exception as e:
if not ignore_errors:
raise
print >> sys.stderr, 'Error: %s' % e
def copy(self, src, dest):
"""Copy the given document to create a new document.
:param src: the ID of the document to copy, or a dictionary or
`Document` object representing the source document.
:param dest: either the destination document ID as string, or a
dictionary or `Document` instance of the document that
should be overwritten.
:return: the new revision of the destination document
:rtype: `str`
:since: 0.6
"""
if not isinstance(src, basestring):
if not isinstance(src, dict):
if hasattr(src, "items"):
src = dict(src.items())
else:
raise TypeError("expected dict or string, got %s" % type(src))
src = src["_id"]
if not isinstance(dest, basestring):
if not isinstance(dest, dict):
if hasattr(dest, "items"):
dest = dict(dest.items())
else:
raise TypeError("expected dict or string, got %s" % type(dest))
if "_rev" in dest:
dest = "%s?%s" % (http.quote(dest["_id"]), http.urlencode({"rev": dest["_rev"]}))
else:
dest = http.quote(dest["_id"])
_, _, data = self.resource._request("COPY", src, headers={"Destination": dest})
data = json.decode(data.read())
return data["rev"]
def load_multipart_file(fn):
fh = file(fn, 'r') #codecs.open(fn, 'r', 'utf-8')
for headers, is_multipart, payload in read_multipart(fh):
doc = cdb_json.decode(payload)
#print "Reading speech", doc.get('speech_id'), "..."
speech = Speech(doc)
yield speech
#speeches.append()
fh.close()
def get_revisions_dict(database, row):
doc = urllib2.urlopen(SERVER + "/" + database + "/" + row['key'] +
"?revs_info=true").read()
doc = json.decode(doc)
result = []
for dictionary in doc.get('_revs_info'):
if dictionary.get('status') == "available":
result.append(dictionary.get('rev'))
return result
def tmp(docIdx):
resp, content = http.request('%s/%s/%s' % (params.server, params.testdb, docs[i]), 'GET')
obj = json.decode(content)
if obj.has_key('counter'):
obj['counter'] = obj['counter'] + 1
else:
obj['counter'] = 0
doc = json.encode(obj)
http.request('%s/%s/%s' % (params.server, params.testdb, docs[i]), 'PUT', body=doc)
def _changes(self, **opts):
_, _, data = self.resource.get('_changes', **opts)
lines = data.iterchunks()
for ln in lines:
if not ln: # skip heartbeats
continue
doc = json.decode(ln.decode('utf-8'))
if 'last_seq' in doc: # consume the rest of the response if this
for ln in lines: # was the last line, allows conn reuse
pass
yield doc
def _changes(self, **opts):
_, _, data = self.resource.get('_changes', **opts)
lines = iter(data)
for ln in lines:
if not ln: # skip heartbeats
continue
doc = json.decode(ln)
if 'last_seq' in doc: # consume the rest of the response if this
for ln in lines: # was the last line, allows conn reuse
pass
yield doc
def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False):
db = Database(dburl)
if username is not None and password is not None:
db.resource.credentials = (username, password)
hostname = Is_Server(dburl, db.name)
for headers, is_multipart, payload in read_multipart(fileobj):
docid = headers['content-id']
time.sleep(1)
if '-db' in db.name:
if 'credential' in docid or 'flow' in docid or 'setting' in docid or 'functions' in docid:
docid = Rename_docid(docid, hostname)
obj = db.get(docid)
if obj == None:
new_doc = {'_id': docid}
db.save(new_doc)
obj = db.get(docid)
if is_multipart: # doc has attachments
for headers, _, payload in payload:
if 'content-id' not in headers:
doc = json.decode(payload)
doc['_attachments'] = {}
else:
doc['_attachments'][headers['content-id']] = {
'data': b64encode(payload).decode('ascii'),
'content_type': headers['content-type'],
'length': len(payload)
}
else: # no attachments, just the JSON
doc = json.decode(payload)
doc['_rev'] = obj['_rev']
doc['_id'] = obj['_id']
print('Loading document %r' % docid, file=sys.stderr)
try:
db[docid] = doc
except Exception as e:
if not ignore_errors:
raise
print('Error: %s' % e, file=sys.stderr)
def _request_json(self,
method,
path=None,
body=None,
headers=None,
**params):
status, headers, data = self._request(method,
path,
body=body,
headers=headers,
**params)
if 'application/json' in headers.get('content-type', ''):
data = json.decode(data.read().decode('utf-8'))
return status, headers, data
def _changes(self, **opts):
opts.setdefault('yield_beats', False)
yield_beats = opts.pop('yield_beats')
_, _, data = self.resource.get('_changes', **opts)
lines = iter(data)
for ln in lines:
if not ln: # skip heartbeats
if yield_beats:
yield {}
continue
doc = json.decode(ln)
if 'last_seq' in doc: # consume the rest of the response if this
for ln in lines: # was the last line, allows conn reuse
pass
yield doc
def _changes(self, **opts):
opts.setdefault('yield_beats', False)
yield_beats = opts.pop('yield_beats')
_, _, data = self.resource.get('_changes', **opts)
lines = iter(data)
for ln in lines:
if not ln: # skip heartbeats
if yield_beats:
yield {}
continue
doc = json.decode(ln)
if 'last_seq' in doc: # consume the rest of the response if this
for ln in lines: # was the last line, allows conn reuse
pass
yield doc
def get_json_raw_url(mb, path):
"""
little workaround for skirting overzealous
couchdb-python quoting behavior.
"""
from couchdb import http, json
session = mb.resource.session
creds = mb.resource.credentials
method = "GET"
path = [mb.resource.url] + path
url = http.urljoin(*path)
status, headers, data = session.request(method, url, credentials=creds)
if "application/json" in headers.get("content-type"):
data = json.decode(data.read())
return status, headers, data
def run(self):
url = self._path()
res = Requester().get(url, headers={'accept': 'application/json'})
res_data = res.read()
res.close()
doc = couchdb_json.decode(res_data)
doc_count = doc['doc_count']
with couchdb_write_multipart(sys.stdout, boundary=None) as envelope:
done = 0
sys.stderr.write("Doc Count: %d\n" % doc_count)
sys.stderr.write("Chunk Size: %d\n" % self._chunk_size)
while done < doc_count:
batch_size = self._run_chunk(envelope, done)
if batch_size == 0:
break
else:
done += batch_size
sys.stderr.write("%.2f%% %d/%d\n" % \
((float(done) / float(doc_count) * 100),
done, doc_count))
sys.stderr.write("\n")
pass
def copy(self, src, dest):
"""Copy the given document to create a new document.
:param src: the ID of the document to copy, or a dictionary or
`Document` object representing the source document.
:param dest: either the destination document ID as string, or a
dictionary or `Document` instance of the document that
should be overwritten.
:return: the new revision of the destination document
:rtype: `str`
:since: 0.6
"""
if not isinstance(src, basestring):
if not isinstance(src, dict):
if hasattr(src, 'items'):
src = dict(src.items())
else:
raise TypeError('expected dict or string, got %s' %
type(src))
src = src['_id']
if not isinstance(dest, basestring):
if not isinstance(dest, dict):
if hasattr(dest, 'items'):
dest = dict(dest.items())
else:
raise TypeError('expected dict or string, got %s' %
type(dest))
if '_rev' in dest:
dest = '%s?%s' % (http.quote(
dest['_id']), http.urlencode({'rev': dest['_rev']}))
else:
dest = http.quote(dest['_id'])
_, _, data = self.resource._request('COPY',
src,
headers={'Destination': dest})
data = json.decode(data.read())
return data['rev']
def __call__(self):
"""Reads notifications from stdin and triggers replication"""
options = self.options
wait_counter = time.time()
while True:
# non-blocking readline(), raises IOErrors
fcntl.fcntl(sys.stdin, fcntl.F_SETFL, os.O_NONBLOCK)
try:
line = sys.stdin.readline()
# poor man's validation. If we get garbage, we sys.exit
if not line.endswith('}\n'):
sys.exit(0)
note = json.decode(line)
log.debug('Received %r', note)
# we don't care for deletes
if note['type'] == 'delete' and not options.ignore_deletes:
continue
self.databases.append((note['type'], note['db']))
# if there are more docs that we want to batch, flush
if len(self.databases) >= int(options.batch_threshold):
self.sync_databases()
continue
except IOError:
# if we waited longer that we want to wait, flush
if (time.time() - wait_counter) > int(options.wait_threshold):
self.sync_databases()
wait_counter = time.time()
time.sleep(float(options.wait_threshold))
def copy(self, src, dest):
"""Copy the given document to create a new document.
:param src: the ID of the document to copy, or a dictionary or
`Document` object representing the source document.
:param dest: either the destination document ID as string, or a
dictionary or `Document` instance of the document that
should be overwritten.
:return: the new revision of the destination document
:rtype: `str`
:since: 0.6
"""
if not isinstance(src, util.strbase):
if not isinstance(src, dict):
if hasattr(src, 'items'):
src = dict(src.items())
else:
raise TypeError('expected dict or string, got %s' %
type(src))
src = src['_id']
if not isinstance(dest, util.strbase):
if not isinstance(dest, dict):
if hasattr(dest, 'items'):
dest = dict(dest.items())
else:
raise TypeError('expected dict or string, got %s' %
type(dest))
if '_rev' in dest:
dest = '%s?%s' % (http.quote(dest['_id']),
http.urlencode({'rev': dest['_rev']}))
else:
dest = http.quote(dest['_id'])
_, _, data = self.resource._request('COPY', src,
headers={'Destination': dest})
data = json.decode(data.read().decode('utf-8'))
return data['rev']
def revsion_map(database, dbm):
results = dbm.database.query(map_form_model_for_subject_questionnaires)
revid_map = {}
for row in results:
revision_ids = get_revisions_dict(database, row)
log_statement("Form Model _id : %s " % row['key'])
form_model = get_form_model(dbm, row['value'])
if is_max_len_equal_to(form_model, 20):
rev_with_20 = form_model.revision
for previous_rev in revision_ids:
revision_doc = urllib2.urlopen(SERVER + "/" + database + "/" +
row['key'] + "?rev=" +
previous_rev).read()
revision_doc = get_form_model(dbm, json.decode(revision_doc))
if is_max_len_equal_to(revision_doc, 12):
revid_map[previous_rev] = (rev_with_20,
form_model.form_code)
log_statement(
"added revision for form:%s old:%s new:%s " %
(form_model.form_code, previous_rev, rev_with_20))
break
rev_with_20 = previous_rev
return revid_map
results = function(keys, vals)
return [True, [results]]
def rereduce(*cmd):
return reduce(*cmd, **{'rereduce': True})
handlers = {'reset': reset, 'add_fun': add_fun, 'map_doc': map_doc,
'reduce': reduce, 'rereduce': rereduce}
try:
while True:
line = input.readline()
if not line:
break
try:
cmd = json.decode(line)
log.debug('Processing %r', cmd)
except ValueError, e:
log.error('Error: %s', e, exc_info=True)
return 1
else:
retval = handlers[cmd[0]](*cmd[1:])
log.debug('Returning %r', retval)
_writejson(retval)
except KeyboardInterrupt:
return 0
except Exception, e:
log.error('Error: %s', e, exc_info=True)
return 1
def request(self,
method,
url,
body=None,
headers=None,
credentials=None,
num_redirects=0):
if url in self.perm_redirects:
url = self.perm_redirects[url]
method = method.upper()
if headers is None:
headers = {}
headers.setdefault('Accept', 'application/json')
headers['User-Agent'] = self.user_agent
cached_resp = None
if method in ('GET', 'HEAD'):
cached_resp = self.cache.get(url)
if cached_resp is not None:
etag = cached_resp[1].get('etag')
if etag:
headers['If-None-Match'] = etag
if (body is not None and not isinstance(body, util.strbase)
and not hasattr(body, 'read')):
body = json.encode(body).encode('utf-8')
headers.setdefault('Content-Type', 'application/json')
if body is None:
headers.setdefault('Content-Length', '0')
elif isinstance(body, util.strbase):
headers.setdefault('Content-Length', str(len(body)))
else:
headers['Transfer-Encoding'] = 'chunked'
authorization = basic_auth(credentials)
if authorization:
headers['Authorization'] = authorization
path_query = util.urlunsplit(('', '') + util.urlsplit(url)[2:4] +
('', ))
conn = self.connection_pool.get(url)
def _try_request_with_retries(retries):
while True:
try:
return _try_request()
except socket.error as e:
ecode = e.args[0]
if ecode not in self.retryable_errors:
raise
try:
delay = next(retries)
except StopIteration:
# No more retries, raise last socket error.
raise e
finally:
time.sleep(delay)
conn.close()
def _try_request():
try:
conn.putrequest(method, path_query, skip_accept_encoding=True)
for header in headers:
conn.putheader(header, headers[header])
if body is None:
conn.endheaders()
else:
if isinstance(body, util.strbase):
if isinstance(body, util.utype):
conn.endheaders(body.encode('utf-8'))
else:
conn.endheaders(body)
else: # assume a file-like object and send in chunks
conn.endheaders()
while 1:
chunk = body.read(CHUNK_SIZE)
if not chunk:
break
if isinstance(chunk, util.utype):
chunk = chunk.encode('utf-8')
status = ('%x\r\n' % len(chunk)).encode('utf-8')
conn.send(status + chunk + b'\r\n')
conn.send(b'0\r\n\r\n')
return conn.getresponse()
except BadStatusLine as e:
# httplib raises a BadStatusLine when it cannot read the status
# line saying, "Presumably, the server closed the connection
# before sending a valid response."
# Raise as ECONNRESET to simplify retry logic.
if e.line == '' or e.line == "''":
raise socket.error(errno.ECONNRESET)
else:
raise
resp = _try_request_with_retries(iter(self.retry_delays))
status = resp.status
# Handle conditional response
if status == 304 and method in ('GET', 'HEAD'):
resp.read()
self.connection_pool.release(url, conn)
status, msg, data = cached_resp
if data is not None:
data = util.StringIO(data)
return status, msg, data
elif cached_resp:
self.cache.remove(url)
# Handle redirects
if status == 303 or \
method in ('GET', 'HEAD') and status in (301, 302, 307):
resp.read()
self.connection_pool.release(url, conn)
if num_redirects > self.max_redirects:
raise RedirectLimit('Redirection limit exceeded')
location = resp.getheader('location')
if status == 301:
self.perm_redirects[url] = location
elif status == 303:
method = 'GET'
return self.request(method,
location,
body,
headers,
num_redirects=num_redirects + 1)
data = None
streamed = False
# Read the full response for empty responses so that the connection is
# in good state for the next request
if method == 'HEAD' or resp.getheader('content-length') == '0' or \
status < 200 or status in (204, 304):
resp.read()
self.connection_pool.release(url, conn)
# Buffer small non-JSON response bodies
elif int(resp.getheader('content-length', sys.maxsize)) < CHUNK_SIZE:
data = resp.read()
self.connection_pool.release(url, conn)
# For large or chunked response bodies, do not buffer the full body,
# and instead return a minimal file-like object
else:
data = ResponseBody(resp, self.connection_pool, url, conn)
streamed = True
# Handle errors
if status >= 400:
ctype = resp.getheader('content-type')
if data is not None and 'application/json' in ctype:
data = json.decode(data.decode('utf-8'))
error = data.get('error'), data.get('reason')
elif method != 'HEAD':
error = resp.read()
self.connection_pool.release(url, conn)
else:
error = ''
if status == 401:
raise Unauthorized(error)
elif status == 404:
raise ResourceNotFound(error)
elif status == 409:
raise ResourceConflict(error)
elif status == 412:
raise PreconditionFailed(error)
else:
raise ServerError((status, error))
# Store cachable responses
if not streamed and method == 'GET' and 'etag' in resp.msg:
self.cache.put(url, (status, resp.msg, data))
if not streamed and data is not None:
data = util.StringIO(data)
return status, resp.msg, data
def run(input=sys.stdin, output=sys.stdout):
r"""CouchDB view function handler implementation for Python.
:param input: the readable file-like object to read input from
:param output: the writable file-like object to write output to
"""
functions = []
def _writejson(obj):
obj = json.encode(obj)
if isinstance(obj, util.utype):
obj = obj.encode('utf-8')
output.write(obj)
output.write(b'\n')
output.flush()
def _log(message):
if not isinstance(message, util.strbase):
message = json.encode(message)
_writejson({'log': message})
def reset(config=None):
del functions[:]
return True
def add_fun(string):
string = BOM_UTF8 + string.encode('utf-8')
globals_ = {}
try:
util.pyexec(string, {'log': _log}, globals_)
except Exception as e:
return {
'error': {
'id': 'map_compilation_error',
'reason': e.args[0]
}
}
err = {
'error': {
'id':
'map_compilation_error',
'reason':
'string must eval to a function '
'(ex: "def(doc): return 1")'
}
}
if len(globals_) != 1:
return err
function = list(globals_.values())[0]
if type(function) is not FunctionType:
return err
functions.append(function)
return True
def map_doc(doc):
results = []
for function in functions:
try:
results.append([[key, value] for key, value in function(doc)])
except Exception as e:
log.error('runtime error in map function: %s',
e,
exc_info=True)
results.append([])
_log(traceback.format_exc())
return results
def reduce(*cmd, **kwargs):
code = BOM_UTF8 + cmd[0][0].encode('utf-8')
args = cmd[1]
globals_ = {}
try:
util.pyexec(code, {'log': _log}, globals_)
except Exception as e:
log.error('runtime error in reduce function: %s', e, exc_info=True)
return {
'error': {
'id': 'reduce_compilation_error',
'reason': e.args[0]
}
}
err = {
'error': {
'id':
'reduce_compilation_error',
'reason':
'string must eval to a function '
'(ex: "def(keys, values): return 1")'
}
}
if len(globals_) != 1:
return err
function = list(globals_.values())[0]
if type(function) is not FunctionType:
return err
rereduce = kwargs.get('rereduce', False)
results = []
if rereduce:
keys = None
vals = args
else:
if args:
keys, vals = zip(*args)
else:
keys, vals = [], []
if util.funcode(function).co_argcount == 3:
results = function(keys, vals, rereduce)
else:
results = function(keys, vals)
return [True, [results]]
def rereduce(*cmd):
# Note: weird kwargs is for Python 2.5 compat
return reduce(*cmd, **{'rereduce': True})
handlers = {
'reset': reset,
'add_fun': add_fun,
'map_doc': map_doc,
'reduce': reduce,
'rereduce': rereduce
}
try:
while True:
line = input.readline()
if not line:
break
try:
cmd = json.decode(line)
log.debug('Processing %r', cmd)
except ValueError as e:
log.error('Error: %s', e, exc_info=True)
return 1
else:
retval = handlers[cmd[0]](*cmd[1:])
log.debug('Returning %r', retval)
_writejson(retval)
except KeyboardInterrupt:
return 0
except Exception as e:
log.error('Error: %s', e, exc_info=True)
return 1
elif int(resp.getheader('content-length', sys.maxint)) < CHUNK_SIZE:
data = resp.read()
self.connection_pool.release(url, conn)
# For large or chunked response bodies, do not buffer the full body,
# and instead return a minimal file-like object
else:
data = ResponseBody(resp,
lambda: self.connection_pool.release(url, conn))
streamed = True
# Handle errors
if status >= 400:
ctype = resp.getheader('content-type')
if data is not None and 'application/json' in ctype:
data = json.decode(data)
error = data.get('error'), data.get('reason')
elif method != 'HEAD':
error = resp.read()
self.connection_pool.release(url, conn)
else:
error = ''
if status == 401:
raise Unauthorized(error)
elif status == 404:
raise ResourceNotFound(error)
elif status == 409:
raise ResourceConflict(error)
elif status == 412:
raise PreconditionFailed(error)
else:
def run(input=sys.stdin, output=sys.stdout):
r"""CouchDB view function handler implementation for Python.
:param input: the readable file-like object to read input from
:param output: the writable file-like object to write output to
"""
functions = []
def _writejson(obj):
obj = json.encode(obj)
if isinstance(obj, util.utype):
obj = obj.encode('utf-8')
output.write(obj)
output.write(b'\n')
output.flush()
def _log(message):
if not isinstance(message, util.strbase):
message = json.encode(message)
_writejson({'log': message})
def reset(config=None):
del functions[:]
return True
def add_fun(string):
string = BOM_UTF8 + string.encode('utf-8')
globals_ = {}
try:
util.pyexec(string, {'log': _log}, globals_)
except Exception as e:
return {'error': {
'id': 'map_compilation_error',
'reason': e.args[0]
}}
err = {'error': {
'id': 'map_compilation_error',
'reason': 'string must eval to a function '
'(ex: "def(doc): return 1")'
}}
if len(globals_) != 1:
return err
function = list(globals_.values())[0]
if type(function) is not FunctionType:
return err
functions.append(function)
return True
def map_doc(doc):
results = []
for function in functions:
try:
results.append([[key, value] for key, value in function(doc)])
except Exception as e:
log.error('runtime error in map function: %s', e,
exc_info=True)
results.append([])
_log(traceback.format_exc())
return results
def reduce(*cmd, **kwargs):
code = BOM_UTF8 + cmd[0][0].encode('utf-8')
args = cmd[1]
globals_ = {}
try:
util.pyexec(code, {'log': _log}, globals_)
except Exception as e:
log.error('runtime error in reduce function: %s', e,
exc_info=True)
return {'error': {
'id': 'reduce_compilation_error',
'reason': e.args[0]
}}
err = {'error': {
'id': 'reduce_compilation_error',
'reason': 'string must eval to a function '
'(ex: "def(keys, values): return 1")'
}}
if len(globals_) != 1:
return err
function = list(globals_.values())[0]
if type(function) is not FunctionType:
return err
rereduce = kwargs.get('rereduce', False)
results = []
if rereduce:
keys = None
vals = args
else:
if args:
keys, vals = zip(*args)
else:
keys, vals = [], []
if util.funcode(function).co_argcount == 3:
results = function(keys, vals, rereduce)
else:
results = function(keys, vals)
return [True, [results]]
def rereduce(*cmd):
# Note: weird kwargs is for Python 2.5 compat
return reduce(*cmd, **{'rereduce': True})
handlers = {'reset': reset, 'add_fun': add_fun, 'map_doc': map_doc,
'reduce': reduce, 'rereduce': rereduce}
try:
while True:
line = input.readline()
if not line:
break
try:
cmd = json.decode(line)
log.debug('Processing %r', cmd)
except ValueError as e:
log.error('Error: %s', e, exc_info=True)
return 1
else:
retval = handlers[cmd[0]](*cmd[1:])
log.debug('Returning %r', retval)
_writejson(retval)
except KeyboardInterrupt:
return 0
except Exception as e:
log.error('Error: %s', e, exc_info=True)
return 1
def put_json(self, *a, **k):
status, headers, data = self.put(*a, **k)
if 'application/json' in headers.get('content-type'):
data = json.decode(data.read())
return status, headers, data
def put_json(self, *a, **k):
status, headers, data = self.put(*a, **k)
if 'application/json' in headers.get('content-type'):
data = json.decode(data.read())
return status, headers, data
elif int(resp.getheader('content-length', sys.maxint)) < CHUNK_SIZE:
data = resp.read()
self._return_connection(url, conn)
# For large or chunked response bodies, do not buffer the full body,
# and instead return a minimal file-like object
else:
data = ResponseBody(resp,
lambda: self._return_connection(url, conn))
streamed = True
# Handle errors
if status >= 400:
ctype = resp.getheader('content-type')
if data is not None and 'application/json' in ctype:
data = json.decode(data)
error = data.get('error'), data.get('reason')
elif method != 'HEAD':
error = resp.read()
self._return_connection(url, conn)
else:
error = ''
if status == 401:
raise Unauthorized(error)
elif status == 404:
raise ResourceNotFound(error)
elif status == 409:
raise ResourceConflict(error)
elif status == 412:
raise PreconditionFailed(error)
else:
def request(self, method, url, body=None, headers=None, credentials=None,
num_redirects=0):
if url in self.perm_redirects:
url = self.perm_redirects[url]
method = method.upper()
if headers is None:
headers = {}
headers.setdefault('Accept', 'application/json')
headers['User-Agent'] = self.user_agent
cached_resp = None
if method in ('GET', 'HEAD'):
cached_resp = self.cache.get(url)
if cached_resp is not None:
etag = cached_resp[1].get('etag')
if etag:
headers['If-None-Match'] = etag
if (body is not None and not isinstance(body, util.strbase) and
not hasattr(body, 'read')):
body = json.encode(body).encode('utf-8')
headers.setdefault('Content-Type', 'application/json')
if body is None:
headers.setdefault('Content-Length', '0')
elif isinstance(body, util.strbase):
headers.setdefault('Content-Length', str(len(body)))
else:
headers['Transfer-Encoding'] = 'chunked'
authorization = basic_auth(credentials)
if authorization:
headers['Authorization'] = authorization
path_query = util.urlunsplit(('', '') + util.urlsplit(url)[2:4] + ('',))
conn = self.connection_pool.get(url)
def _try_request_with_retries(retries):
while True:
try:
return _try_request()
except socket.error as e:
ecode = e.args[0]
if ecode not in self.retryable_errors:
raise
try:
delay = retries.next()
except StopIteration:
# No more retries, raise last socket error.
raise e
time.sleep(delay)
conn.close()
def _try_request():
try:
conn.putrequest(method, path_query, skip_accept_encoding=True)
for header in headers:
conn.putheader(header, headers[header])
if body is None:
conn.endheaders()
else:
if isinstance(body, util.strbase):
if isinstance(body, util.utype):
conn.endheaders(body.encode('utf-8'))
else:
conn.endheaders(body)
else: # assume a file-like object and send in chunks
conn.endheaders()
while 1:
chunk = body.read(CHUNK_SIZE)
if not chunk:
break
if isinstance(chunk, util.utype):
chunk = chunk.encode('utf-8')
status = ('%x\r\n' % len(chunk)).encode('utf-8')
conn.send(status + chunk + b'\r\n')
conn.send(b'0\r\n\r\n')
return conn.getresponse()
except BadStatusLine as e:
# httplib raises a BadStatusLine when it cannot read the status
# line saying, "Presumably, the server closed the connection
# before sending a valid response."
# Raise as ECONNRESET to simplify retry logic.
if e.line == '' or e.line == "''":
raise socket.error(errno.ECONNRESET)
else:
raise
resp = _try_request_with_retries(iter(self.retry_delays))
status = resp.status
# Handle conditional response
if status == 304 and method in ('GET', 'HEAD'):
resp.read()
self.connection_pool.release(url, conn)
status, msg, data = cached_resp
if data is not None:
data = util.StringIO(data)
return status, msg, data
elif cached_resp:
self.cache.remove(url)
# Handle redirects
if status == 303 or \
method in ('GET', 'HEAD') and status in (301, 302, 307):
resp.read()
self.connection_pool.release(url, conn)
if num_redirects > self.max_redirects:
raise RedirectLimit('Redirection limit exceeded')
location = resp.getheader('location')
if status == 301:
self.perm_redirects[url] = location
elif status == 303:
method = 'GET'
return self.request(method, location, body, headers,
num_redirects=num_redirects + 1)
data = None
streamed = False
# Read the full response for empty responses so that the connection is
# in good state for the next request
if method == 'HEAD' or resp.getheader('content-length') == '0' or \
status < 200 or status in (204, 304):
resp.read()
self.connection_pool.release(url, conn)
# Buffer small non-JSON response bodies
elif int(resp.getheader('content-length', sys.maxsize)) < CHUNK_SIZE:
data = resp.read()
self.connection_pool.release(url, conn)
# For large or chunked response bodies, do not buffer the full body,
# and instead return a minimal file-like object
else:
data = ResponseBody(resp, self.connection_pool, url, conn)
streamed = True
# Handle errors
if status >= 400:
ctype = resp.getheader('content-type')
if data is not None and 'application/json' in ctype:
data = json.decode(data.decode('utf-8'))
error = data.get('error'), data.get('reason')
elif method != 'HEAD':
error = resp.read()
self.connection_pool.release(url, conn)
else:
error = ''
if status == 401:
raise Unauthorized(error)
elif status == 404:
raise ResourceNotFound(error)
elif status == 409:
raise ResourceConflict(error)
elif status == 412:
raise PreconditionFailed(error)
else:
raise ServerError((status, error))
# Store cachable responses
if not streamed and method == 'GET' and 'etag' in resp.msg:
self.cache.put(url, (status, resp.msg, data))
if not streamed and data is not None:
data = util.StringIO(data)
return status, resp.msg, data
def _request_json(self, method, path=None, body=None, headers=None, **params):
status, headers, data = self._request(method, path, body=body, headers=headers, **params)
if "application/json" in headers.get("content-type"):
data = json.decode(data.read())
return status, headers, data
return [True, [results]]
def rereduce(*cmd):
# Note: weird kwargs is for Python 2.5 compat
return reduce(*cmd, **{'rereduce': True})
handlers = {'reset': reset, 'add_fun': add_fun, 'map_doc': map_doc,
'reduce': reduce, 'rereduce': rereduce}
try:
while True:
line = input.readline()
if not line:
break
try:
cmd = json.decode(line)
log.debug('Processing %r', cmd)
except ValueError, e:
log.error('Error: %s', e, exc_info=True)
return 1
else:
retval = handlers[cmd[0]](*cmd[1:])
log.debug('Returning %r', retval)
_writejson(retval)
except KeyboardInterrupt:
return 0
except Exception, e:
log.error('Error: %s', e, exc_info=True)
return 1
def _request_json(self, method, path=None, body=None, headers=None, **params):
status, headers, data = self._request(method, path, body=body,
headers=headers, **params)
if 'application/json' in headers.get('content-type', ''):
data = json.decode(data.read().decode('utf-8'))
return status, headers, data
class Resource(object):
def __init__(self, http, uri):
if http is None:
http = httplib2.Http()
http.force_exception_to_status_code = False
self.http = http
self.uri = uri
def __call__(self, path):
return type(self)(self.http, uri(self.uri, path))
def delete(self, path=None, headers=None, **params):
return self._request('DELETE', path, headers=headers, **params)
def get(self, path=None, headers=None, **params):
return self._request('GET', path, headers=headers, **params)
def head(self, path=None, headers=None, **params):
return self._request('HEAD', path, headers=headers, **params)
def post(self, path=None, content=None, headers=None, **params):
return self._request('POST', path, content=content, headers=headers,
**params)
def put(self, path=None, content=None, headers=None, **params):
return self._request('PUT', path, content=content, headers=headers,
**params)
def _request(self, method, path=None, content=None, headers=None,
**params):
from couchdb import __version__
headers = headers or {}
headers.setdefault('Accept', 'application/json')
headers.setdefault('User-Agent', 'couchdb-python %s' % __version__)
body = None
if content is not None:
if not isinstance(content, basestring):
body = json.encode(content).encode('utf-8')
headers.setdefault('Content-Type', 'application/json')
else:
body = content
headers.setdefault('Content-Length', str(len(body)))
def _make_request(retry=1):
try:
return self.http.request(uri(self.uri, path, **params), method,
body=body, headers=headers)
except socket.error, e:
if retry > 0 and e.args[0] == 54: # reset by peer
return _make_request(retry - 1)
raise
resp, data = _make_request()
status_code = int(resp.status)
if data and resp.get('content-type') == 'application/json':
try:
data = json.decode(data)
except ValueError:
pass
if status_code >= 400:
if type(data) is dict:
error = (data.get('error'), data.get('reason'))
else:
error = data
if status_code == 404:
raise ResourceNotFound(error)
elif status_code == 409:
raise ResourceConflict(error)
elif status_code == 412:
raise PreconditionFailed(error)
else:
raise ServerError((status_code, error))
return resp, data
