def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None):
parts = data.splitlines() # one at a time
if not parts or parts[0] != '[':
raise BrokenSyncStream
data = parts[1:-1]
comma = False
if data:
line, comma = utils.check_and_strip_comma(data[0])
res = json.loads(line)
if ensure_callback and 'replica_uid' in res:
ensure_callback(res['replica_uid'])
for entry in data[1:]:
if not comma: # missing in between comma
raise BrokenSyncStream
line, comma = utils.check_and_strip_comma(entry)
entry = json.loads(line)
doc = Document(entry['id'], entry['rev'], entry['content'])
return_doc_cb(doc, entry['gen'], entry['trans_id'])
if parts[-1] != ']':
try:
partdic = json.loads(parts[-1])
except ValueError:
pass
else:
if isinstance(partdic, dict):
self._error(partdic)
raise BrokenSyncStream
if not data or comma: # no entries or bad extra comma
raise BrokenSyncStream
return res
def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None):
parts = data.splitlines() # one at a time
if not parts or parts[0] != '[':
raise BrokenSyncStream
data = parts[1:-1]
comma = False
if data:
line, comma = utils.check_and_strip_comma(data[0])
res = json.loads(line)
if ensure_callback and 'replica_uid' in res:
ensure_callback(res['replica_uid'])
for entry in data[1:]:
if not comma: # missing in between comma
raise BrokenSyncStream
line, comma = utils.check_and_strip_comma(entry)
entry = json.loads(line)
doc = Document(entry['id'], entry['rev'], entry['content'])
return_doc_cb(doc, entry['gen'], entry['trans_id'])
if parts[-1] != ']':
try:
partdic = json.loads(parts[-1])
except ValueError:
pass
else:
if isinstance(partdic, dict):
self._error(partdic)
raise BrokenSyncStream
if not data or comma: # no entries or bad extra comma
raise BrokenSyncStream
return res
def __call__(self):
args = urlparse.parse_qsl(self.environ['QUERY_STRING'],
strict_parsing=False)
try:
args = dict(
(k.decode('utf-8'), v.decode('utf-8')) for k, v in args)
except ValueError:
raise BadRequest()
method = self.environ['REQUEST_METHOD'].lower()
if method in ('get', 'delete'):
meth = self._lookup(method)
return meth(args, None)
else:
# we expect content-length > 0, reconsider if we move
# to support chunked enconding
try:
content_length = int(self.environ['CONTENT_LENGTH'])
except (ValueError, KeyError):
raise BadRequest
if content_length <= 0:
raise BadRequest
if content_length > self.max_request_size:
raise BadRequest
reader = _FencedReader(self.environ['wsgi.input'], content_length,
self.max_entry_size)
content_type = self.environ.get('CONTENT_TYPE', '')
content_type = content_type.split(';', 1)[0].strip()
if content_type == 'application/json':
meth = self._lookup(method)
body = reader.read_chunk(sys.maxint)
return meth(args, body)
elif content_type == 'application/x-u1db-sync-stream':
meth_args = self._lookup('%s_args' % method)
meth_entry = self._lookup('%s_stream_entry' % method)
meth_end = self._lookup('%s_end' % method)
body_getline = reader.getline
if body_getline().strip() != '[':
raise BadRequest()
line = body_getline()
line, comma = utils.check_and_strip_comma(line.strip())
meth_args(args, line)
while True:
line = body_getline()
entry = line.strip()
if entry == ']':
break
if not entry or not comma: # empty or no prec comma
raise BadRequest
entry, comma = utils.check_and_strip_comma(entry)
meth_entry({}, entry)
if comma or body_getline(): # extra comma or data
raise BadRequest
return meth_end()
else:
raise BadRequest()
def __call__(self):
args = urlparse.parse_qsl(self.environ['QUERY_STRING'],
strict_parsing=False)
try:
args = dict(
(k.decode('utf-8'), v.decode('utf-8')) for k, v in args)
except ValueError:
raise BadRequest()
method = self.environ['REQUEST_METHOD'].lower()
if method in ('get', 'delete'):
meth = self._lookup(method)
return meth(args, None)
else:
# we expect content-length > 0, reconsider if we move
# to support chunked enconding
try:
content_length = int(self.environ['CONTENT_LENGTH'])
except (ValueError, KeyError):
raise BadRequest
if content_length <= 0:
raise BadRequest
if content_length > self.max_request_size:
raise BadRequest
reader = _FencedReader(self.environ['wsgi.input'], content_length,
self.max_entry_size)
content_type = self.environ.get('CONTENT_TYPE', '')
content_type = content_type.split(';', 1)[0].strip()
if content_type == 'application/json':
meth = self._lookup(method)
body = reader.read_chunk(sys.maxint)
return meth(args, body)
elif content_type == 'application/x-u1db-sync-stream':
meth_args = self._lookup('%s_args' % method)
meth_entry = self._lookup('%s_stream_entry' % method)
meth_end = self._lookup('%s_end' % method)
body_getline = reader.getline
if body_getline().strip() != '[':
raise BadRequest()
line = body_getline()
line, comma = utils.check_and_strip_comma(line.strip())
meth_args(args, line)
while True:
line = body_getline()
entry = line.strip()
if entry == ']':
break
if not entry or not comma: # empty or no prec comma
raise BadRequest
entry, comma = utils.check_and_strip_comma(entry)
meth_entry({}, entry)
if comma or body_getline(): # extra comma or data
raise BadRequest
return meth_end()
else:
raise BadRequest()
def dataReceived(self, data):
"""
Buffer incoming data until a line breaks comes in. We check only
the incoming data for efficiency.
"""
self._buffer.write(data)
if '\n' not in data:
return
lines = self.consumeBufferLines()
while lines:
line, _ = utils.check_and_strip_comma(lines.pop(0))
self.lineReceived(line)
self._line += 1
def dataReceived(self, data):
"""
Buffer incoming data until a line breaks comes in. We check only
the incoming data for efficiency.
"""
self._buffer.write(data)
if '\n' not in data:
return
lines = self.consumeBufferLines()
while lines:
line, _ = utils.check_and_strip_comma(lines.pop(0))
self.lineReceived(line)
self._line += 1
def _parse_received_doc_response(self, response):
"""
Parse the response from the server containing the received document.
:param response: The body and headers of the response.
:type response: tuple(str, dict)
:return: (new_gen, new_trans_id, number_of_changes, doc_id, rev,
content, gen, trans_id)
:rtype: tuple
"""
# decode incoming stream
parts = response.splitlines()
if not parts or parts[0] != '[' or parts[-1] != ']':
raise errors.BrokenSyncStream
data = parts[1:-1]
# decode metadata
try:
line, comma = utils.check_and_strip_comma(data[0])
metadata = None
except (IndexError):
raise errors.BrokenSyncStream
try:
metadata = json.loads(line)
new_generation = metadata['new_generation']
new_transaction_id = metadata['new_transaction_id']
number_of_changes = metadata['number_of_changes']
except (ValueError, KeyError):
raise errors.BrokenSyncStream
# make sure we have replica_uid from fresh new dbs
if self._ensure_callback and 'replica_uid' in metadata:
self._ensure_callback(metadata['replica_uid'])
# parse incoming document info
doc_id = None
rev = None
content = None
gen = None
trans_id = None
if number_of_changes > 0:
try:
entry = json.loads(data[1])
doc_id = entry['id']
rev = entry['rev']
content = entry['content']
gen = entry['gen']
trans_id = entry['trans_id']
except (IndexError, KeyError):
raise errors.BrokenSyncStream
return new_generation, new_transaction_id, number_of_changes, \
doc_id, rev, content, gen, trans_id
def __call__(self):
"""
Call an HTTP method of a resource.
This method was rewritten to allow for a sync flow which uses one POST
request for each transferred document (back and forth).
Usual U1DB sync process transfers all documents from client to server
and back in only one POST request. This is inconvenient for some
reasons, as lack of possibility of gracefully interrupting the sync
process, and possible timeouts for when dealing with large documents
that have to be retrieved and encrypted/decrypted. Because of those,
we split the sync process into many POST requests.
"""
args = urlparse.parse_qsl(self.environ['QUERY_STRING'],
strict_parsing=False)
try:
args = dict(
(k.decode('utf-8'), v.decode('utf-8')) for k, v in args)
except ValueError:
raise http_app.BadRequest()
method = self.environ['REQUEST_METHOD'].lower()
if method in ('get', 'delete'):
meth = self._lookup(method)
return meth(args, None)
else:
# we expect content-length > 0, reconsider if we move
# to support chunked enconding
try:
content_length = int(self.environ['CONTENT_LENGTH'])
except (ValueError, KeyError):
# raise http_app.BadRequest
content_length = self.max_request_size
if content_length <= 0:
raise http_app.BadRequest
if content_length > self.max_request_size:
raise http_app.BadRequest
reader = http_app._FencedReader(
self.environ['wsgi.input'], content_length,
self.max_entry_size)
content_type = self.environ.get('CONTENT_TYPE')
if content_type == 'application/json':
meth = self._lookup(method)
body = reader.read_chunk(sys.maxint)
return meth(args, body)
elif content_type.startswith('application/x-soledad-sync'):
# read one line and validate it
body_getline = reader.getline
if body_getline().strip() != '[':
raise http_app.BadRequest()
line = body_getline()
line, comma = utils.check_and_strip_comma(line.strip())
meth_args = self._lookup('%s_args' % method)
meth_args(args, line)
# handle incoming documents
if content_type == 'application/x-soledad-sync-put':
meth_put = self._lookup('%s_put' % method)
meth_end = self._lookup('%s_end' % method)
while True:
entry = body_getline().strip()
if entry == ']': # end of incoming document stream
break
if not entry or not comma: # empty or no prec comma
raise http_app.BadRequest
entry, comma = utils.check_and_strip_comma(entry)
content = body_getline().strip()
content, comma = utils.check_and_strip_comma(content)
meth_put({'content': content or None}, entry)
if comma or body_getline(): # extra comma or data
raise http_app.BadRequest
return meth_end()
# handle outgoing documents
elif content_type == 'application/x-soledad-sync-get':
meth_get = self._lookup('%s_get' % method)
return meth_get()
else:
raise http_app.BadRequest()
else:
raise http_app.BadRequest()
def __call__(self):
"""
Call an HTTP method of a resource.
This method was rewritten to allow for a sync flow which uses one POST
request for each transferred document (back and forth).
Usual U1DB sync process transfers all documents from client to server
and back in only one POST request. This is inconvenient for some
reasons, as lack of possibility of gracefully interrupting the sync
process, and possible timeouts for when dealing with large documents
that have to be retrieved and encrypted/decrypted. Because of those,
we split the sync process into many POST requests.
"""
args = urlparse.parse_qsl(self.environ['QUERY_STRING'],
strict_parsing=False)
try:
args = dict(
(k.decode('utf-8'), v.decode('utf-8')) for k, v in args)
except ValueError:
raise http_app.BadRequest()
method = self.environ['REQUEST_METHOD'].lower()
if method in ('get', 'delete'):
meth = self._lookup(method)
return meth(args, None)
else:
# we expect content-length > 0, reconsider if we move
# to support chunked enconding
try:
content_length = int(self.environ['CONTENT_LENGTH'])
except (ValueError, KeyError):
# raise http_app.BadRequest
content_length = self.max_request_size
if content_length <= 0:
raise http_app.BadRequest
if content_length > self.max_request_size:
raise http_app.BadRequest
reader = http_app._FencedReader(self.environ['wsgi.input'],
content_length,
self.max_entry_size)
content_type = self.environ.get('CONTENT_TYPE')
if content_type == 'application/json':
meth = self._lookup(method)
body = reader.read_chunk(sys.maxint)
return meth(args, body)
elif content_type.startswith('application/x-soledad-sync'):
# read one line and validate it
body_getline = reader.getline
if body_getline().strip() != '[':
raise http_app.BadRequest()
line = body_getline()
line, comma = utils.check_and_strip_comma(line.strip())
meth_args = self._lookup('%s_args' % method)
meth_args(args, line)
# handle incoming documents
if content_type == 'application/x-soledad-sync-put':
meth_put = self._lookup('%s_put' % method)
meth_end = self._lookup('%s_end' % method)
while True:
entry = body_getline().strip()
if entry == ']': # end of incoming document stream
break
if not entry or not comma: # empty or no prec comma
raise http_app.BadRequest
entry, comma = utils.check_and_strip_comma(entry)
content = body_getline().strip()
content, comma = utils.check_and_strip_comma(content)
meth_put({'content': content or None}, entry)
if comma or body_getline(): # extra comma or data
raise http_app.BadRequest
return meth_end()
# handle outgoing documents
elif content_type == 'application/x-soledad-sync-get':
meth_get = self._lookup('%s_get' % method)
return meth_get()
else:
raise http_app.BadRequest()
else:
raise http_app.BadRequest()