def _parse_sync_stream(self, data, return_doc_cb):
parts = data.splitlines() # one at a time
if not parts or parts[0] != '[':
raise BrokenSyncStream
data = parts[1:-1]
if data:
line, comma = utils.check_and_strip_comma(data[0])
res = simplejson.loads(line)
for entry in data[1:]:
if not comma: # missing in between comma
raise BrokenSyncStream
line, comma = utils.check_and_strip_comma(entry)
entry = simplejson.loads(line)
doc = Document(entry['id'], entry['rev'], entry['content'])
return_doc_cb(doc, entry['gen'], entry['trans_id'])
if parts[-1] != ']':
try:
partdic = simplejson.loads(parts[-1])
except ValueError:
pass
else:
if isinstance(partdic, dict):
self._error(partdic)
raise BrokenSyncStream
if comma: # bad extra comma
raise BrokenSyncStream
return res
def _parse_sync_stream(self, data, return_doc_cb):
parts = data.splitlines() # one at a time
if not parts or parts[0] != '[':
raise BrokenSyncStream
data = parts[1:-1]
if data:
line, comma = utils.check_and_strip_comma(data[0])
res = simplejson.loads(line)
for entry in data[1:]:
if not comma: # missing in between comma
raise BrokenSyncStream
line, comma = utils.check_and_strip_comma(entry)
entry = simplejson.loads(line)
doc = Document(entry['id'], entry['rev'], entry['content'])
return_doc_cb(doc, entry['gen'], entry['trans_id'])
if parts[-1] != ']':
try:
partdic = simplejson.loads(parts[-1])
except ValueError:
pass
else:
if isinstance(partdic, dict):
self._error(partdic)
raise BrokenSyncStream
if comma: # bad extra comma
raise BrokenSyncStream
return res
def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None):
parts = data.splitlines() # one at a time
if not parts or parts[0] != '[':
raise BrokenSyncStream
data = parts[1:-1]
comma = False
if data:
line, comma = utils.check_and_strip_comma(data[0])
res = json.loads(line)
if ensure_callback and 'replica_uid' in res:
ensure_callback(res['replica_uid'])
for entry in data[1:]:
if not comma: # missing in between comma
raise BrokenSyncStream
line, comma = utils.check_and_strip_comma(entry)
entry = json.loads(line)
doc = Document(entry['id'], entry['rev'], entry['content'])
return_doc_cb(doc, entry['gen'], entry['trans_id'])
if parts[-1] != ']':
try:
partdic = json.loads(parts[-1])
except ValueError:
pass
else:
if isinstance(partdic, dict):
self._error(partdic)
raise BrokenSyncStream
if not data or comma: # no entries or bad extra comma
raise BrokenSyncStream
return res
def __call__(self):
args = urlparse.parse_qsl(self.environ['QUERY_STRING'],
strict_parsing=False)
try:
args = dict(
(k.decode('utf-8'), v.decode('utf-8')) for k, v in args)
except ValueError:
raise BadRequest()
method = self.environ['REQUEST_METHOD'].lower()
if method in ('get', 'delete'):
meth = self._lookup(method)
return meth(args, None)
else:
# we expect content-length > 0, reconsider if we move
# to support chunked enconding
try:
content_length = int(self.environ['CONTENT_LENGTH'])
except (ValueError, KeyError):
raise BadRequest
if content_length <= 0:
raise BadRequest
if content_length > self.max_request_size:
raise BadRequest
reader = _FencedReader(self.environ['wsgi.input'], content_length,
self.max_entry_size)
content_type = self.environ.get('CONTENT_TYPE', '')
content_type = content_type.split(';', 1)[0].strip()
if content_type == 'application/json':
meth = self._lookup(method)
body = reader.read_chunk(sys.maxint)
return meth(args, body)
elif content_type == 'application/x-u1db-sync-stream':
meth_args = self._lookup('%s_args' % method)
meth_entry = self._lookup('%s_stream_entry' % method)
meth_end = self._lookup('%s_end' % method)
body_getline = reader.getline
if body_getline().strip() != '[':
raise BadRequest()
line = body_getline()
line, comma = utils.check_and_strip_comma(line.strip())
meth_args(args, line)
while True:
line = body_getline()
entry = line.strip()
if entry == ']':
break
if not entry or not comma: # empty or no prec comma
raise BadRequest
entry, comma = utils.check_and_strip_comma(entry)
meth_entry({}, entry)
if comma or body_getline(): # extra comma or data
raise BadRequest
return meth_end()
else:
raise BadRequest()
def __call__(self):
args = urlparse.parse_qsl(self.environ['QUERY_STRING'],
strict_parsing=False)
try:
args = dict(
(k.decode('utf-8'), v.decode('utf-8')) for k, v in args)
except ValueError:
raise BadRequest()
method = self.environ['REQUEST_METHOD'].lower()
if method in ('get', 'delete'):
meth = self._lookup(method)
return meth(args, None)
else:
# we expect content-length > 0, reconsider if we move
# to support chunked enconding
try:
content_length = int(self.environ['CONTENT_LENGTH'])
except (ValueError, KeyError):
raise BadRequest
if content_length <= 0:
raise BadRequest
if content_length > self.max_request_size:
raise BadRequest
reader = _FencedReader(self.environ['wsgi.input'], content_length,
self.max_entry_size)
content_type = self.environ.get('CONTENT_TYPE')
if content_type == 'application/json':
meth = self._lookup(method)
body = reader.read_chunk(sys.maxint)
return meth(args, body)
elif content_type == 'application/x-u1db-sync-stream':
meth_args = self._lookup('%s_args' % method)
meth_entry = self._lookup('%s_stream_entry' % method)
meth_end = self._lookup('%s_end' % method)
body_getline = reader.getline
if body_getline().strip() != '[':
raise BadRequest()
line = body_getline()
line, comma = utils.check_and_strip_comma(line.strip())
meth_args(args, line)
while True:
line = body_getline()
entry = line.strip()
if entry == ']':
break
if not entry or not comma: # empty or no prec comma
raise BadRequest
entry, comma = utils.check_and_strip_comma(entry)
meth_entry({}, entry)
if comma or body_getline(): # extra comma or data
raise BadRequest
return meth_end()
else:
raise BadRequest()
def test_check_and_strip_comma(self):
line, comma = utils.check_and_strip_comma("abc,")
self.assertTrue(comma)
self.assertEqual("abc", line)
line, comma = utils.check_and_strip_comma("abc")
self.assertFalse(comma)
self.assertEqual("abc", line)
line, comma = utils.check_and_strip_comma("")
self.assertFalse(comma)
self.assertEqual("", line)
def test_check_and_strip_comma(self):
line, comma = utils.check_and_strip_comma("abc,")
self.assertTrue(comma)
self.assertEqual("abc", line)
line, comma = utils.check_and_strip_comma("abc")
self.assertFalse(comma)
self.assertEqual("abc", line)
line, comma = utils.check_and_strip_comma("")
self.assertFalse(comma)
self.assertEqual("", line)
def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None):
"""
Does the same as parent's method but ensures incoming content will be
decrypted.
"""
parts = data.splitlines() # one at a time
if not parts or parts[0] != '[':
raise BrokenSyncStream
data = parts[1:-1]
comma = False
if data:
line, comma = utils.check_and_strip_comma(data[0])
res = json.loads(line)
if ensure_callback and 'replica_uid' in res:
ensure_callback(res['replica_uid'])
for entry in data[1:]:
if not comma: # missing in between comma
raise BrokenSyncStream
line, comma = utils.check_and_strip_comma(entry)
entry = json.loads(line)
# decrypt after receiving from server.
if not self._soledad:
raise NoSoledadInstance()
enc_json = json.loads(entry['content'])['_encrypted_json']
if not self._soledad.is_encrypted_sym(enc_json):
raise DocumentNotEncrypted(
"Incoming document from sync is not encrypted.")
doc = LeapDocument(entry['id'],
entry['rev'],
encrypted_json=entry['content'],
soledad=self._soledad)
return_doc_cb(doc, entry['gen'], entry['trans_id'])
if parts[-1] != ']':
try:
partdic = json.loads(parts[-1])
except ValueError:
pass
else:
if isinstance(partdic, dict):
self._error(partdic)
raise BrokenSyncStream
if not data or comma: # no entries or bad extra comma
raise BrokenSyncStream
return res
def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None):
"""
Does the same as parent's method but ensures incoming content will be
decrypted.
"""
parts = data.splitlines() # one at a time
if not parts or parts[0] != '[':
raise BrokenSyncStream
data = parts[1:-1]
comma = False
if data:
line, comma = utils.check_and_strip_comma(data[0])
res = json.loads(line)
if ensure_callback and 'replica_uid' in res:
ensure_callback(res['replica_uid'])
for entry in data[1:]:
if not comma: # missing in between comma
raise BrokenSyncStream
line, comma = utils.check_and_strip_comma(entry)
entry = json.loads(line)
# decrypt after receiving from server.
if not self._soledad:
raise NoSoledadInstance()
enc_json = json.loads(entry['content'])['_encrypted_json']
if not self._soledad.is_encrypted_sym(enc_json):
raise DocumentNotEncrypted(
"Incoming document from sync is not encrypted.")
doc = LeapDocument(entry['id'], entry['rev'],
encrypted_json=entry['content'],
soledad=self._soledad)
return_doc_cb(doc, entry['gen'], entry['trans_id'])
if parts[-1] != ']':
try:
partdic = json.loads(parts[-1])
except ValueError:
pass
else:
if isinstance(partdic, dict):
self._error(partdic)
raise BrokenSyncStream
if not data or comma: # no entries or bad extra comma
raise BrokenSyncStream
return res
def _parse_received_doc_response(self, response):
"""
Parse the response from the server containing the received document.
:param response: The body and headers of the response.
:type response: tuple(str, dict)
:return: (new_gen, new_trans_id, number_of_changes, doc_id, rev,
content, gen, trans_id)
:rtype: tuple
"""
# decode incoming stream
parts = response.splitlines()
if not parts or parts[0] != '[' or parts[-1] != ']':
raise errors.BrokenSyncStream
data = parts[1:-1]
# decode metadata
try:
line, comma = utils.check_and_strip_comma(data[0])
metadata = None
except (IndexError):
raise errors.BrokenSyncStream
try:
metadata = json.loads(line)
new_generation = metadata['new_generation']
new_transaction_id = metadata['new_transaction_id']
number_of_changes = metadata['number_of_changes']
except (ValueError, KeyError):
raise errors.BrokenSyncStream
# make sure we have replica_uid from fresh new dbs
if self._ensure_callback and 'replica_uid' in metadata:
self._ensure_callback(metadata['replica_uid'])
# parse incoming document info
doc_id = None
rev = None
content = None
gen = None
trans_id = None
if number_of_changes > 0:
try:
entry = json.loads(data[1])
doc_id = entry['id']
rev = entry['rev']
content = entry['content']
gen = entry['gen']
trans_id = entry['trans_id']
except (IndexError, KeyError):
raise errors.BrokenSyncStream
return new_generation, new_transaction_id, number_of_changes, \
doc_id, rev, content, gen, trans_id
def _parse_received_doc_response(self, response):
"""
Parse the response from the server containing the received document.
:param response: The body and headers of the response.
:type response: tuple(str, dict)
:return: (new_gen, new_trans_id, number_of_changes, doc_id, rev,
content, gen, trans_id)
:rtype: tuple
"""
# decode incoming stream
parts = response.splitlines()
if not parts or parts[0] != '[' or parts[-1] != ']':
raise errors.BrokenSyncStream
data = parts[1:-1]
# decode metadata
try:
line, comma = utils.check_and_strip_comma(data[0])
metadata = None
except (IndexError):
raise errors.BrokenSyncStream
try:
metadata = json.loads(line)
new_generation = metadata['new_generation']
new_transaction_id = metadata['new_transaction_id']
number_of_changes = metadata['number_of_changes']
except (ValueError, KeyError):
raise errors.BrokenSyncStream
# make sure we have replica_uid from fresh new dbs
if self._ensure_callback and 'replica_uid' in metadata:
self._ensure_callback(metadata['replica_uid'])
# parse incoming document info
doc_id = None
rev = None
content = None
gen = None
trans_id = None
if number_of_changes > 0:
try:
entry = json.loads(data[1])
doc_id = entry['id']
rev = entry['rev']
content = entry['content']
gen = entry['gen']
trans_id = entry['trans_id']
except (IndexError, KeyError):
raise errors.BrokenSyncStream
return new_generation, new_transaction_id, number_of_changes, \
doc_id, rev, content, gen, trans_id
def __call__(self):
"""
Call an HTTP method of a resource.
This method was rewritten to allow for a sync flow which uses one POST
request for each transferred document (back and forth).
Usual U1DB sync process transfers all documents from client to server
and back in only one POST request. This is inconvenient for some
reasons, as lack of possibility of gracefully interrupting the sync
process, and possible timeouts for when dealing with large documents
that have to be retrieved and encrypted/decrypted. Because of those,
we split the sync process into many POST requests.
"""
args = urlparse.parse_qsl(self.environ['QUERY_STRING'],
strict_parsing=False)
try:
args = dict(
(k.decode('utf-8'), v.decode('utf-8')) for k, v in args)
except ValueError:
raise http_app.BadRequest()
method = self.environ['REQUEST_METHOD'].lower()
if method in ('get', 'delete'):
meth = self._lookup(method)
return meth(args, None)
else:
# we expect content-length > 0, reconsider if we move
# to support chunked enconding
try:
content_length = int(self.environ['CONTENT_LENGTH'])
except (ValueError, KeyError):
raise http_app.BadRequest
if content_length <= 0:
raise http_app.BadRequest
if content_length > self.max_request_size:
raise http_app.BadRequest
reader = http_app._FencedReader(
self.environ['wsgi.input'], content_length,
self.max_entry_size)
content_type = self.environ.get('CONTENT_TYPE')
if content_type == 'application/json':
meth = self._lookup(method)
body = reader.read_chunk(sys.maxint)
return meth(args, body)
elif content_type.startswith('application/x-soledad-sync'):
# read one line and validate it
body_getline = reader.getline
if body_getline().strip() != '[':
raise http_app.BadRequest()
line = body_getline()
line, comma = utils.check_and_strip_comma(line.strip())
meth_args = self._lookup('%s_args' % method)
meth_args(args, line)
# handle incoming documents
if content_type == 'application/x-soledad-sync-put':
meth_put = self._lookup('%s_put' % method)
meth_end = self._lookup('%s_end' % method)
entries = []
while True:
line = body_getline()
entry = line.strip()
if entry == ']': # end of incoming document stream
break
if not entry or not comma: # empty or no prec comma
raise http_app.BadRequest
entry, comma = utils.check_and_strip_comma(entry)
entries.append(entry)
if comma or body_getline(): # extra comma or data
raise http_app.BadRequest
for entry in entries:
meth_put({}, entry)
return meth_end()
# handle outgoing documents
elif content_type == 'application/x-soledad-sync-get':
line = body_getline()
entry = line.strip()
meth_get = self._lookup('%s_get' % method)
return meth_get({}, line)
else:
raise http_app.BadRequest()
else:
raise http_app.BadRequest()
def __call__(self):
"""
Call an HTTP method of a resource.
This method was rewritten to allow for a sync flow which uses one POST
request for each transferred document (back and forth).
Usual U1DB sync process transfers all documents from client to server
and back in only one POST request. This is inconvenient for some
reasons, as lack of possibility of gracefully interrupting the sync
process, and possible timeouts for when dealing with large documents
that have to be retrieved and encrypted/decrypted. Because of those,
we split the sync process into many POST requests.
"""
args = urlparse.parse_qsl(self.environ['QUERY_STRING'],
strict_parsing=False)
try:
args = dict(
(k.decode('utf-8'), v.decode('utf-8')) for k, v in args)
except ValueError:
raise http_app.BadRequest()
method = self.environ['REQUEST_METHOD'].lower()
if method in ('get', 'delete'):
meth = self._lookup(method)
return meth(args, None)
else:
# we expect content-length > 0, reconsider if we move
# to support chunked enconding
try:
content_length = int(self.environ['CONTENT_LENGTH'])
except (ValueError, KeyError):
raise http_app.BadRequest
if content_length <= 0:
raise http_app.BadRequest
if content_length > self.max_request_size:
raise http_app.BadRequest
reader = http_app._FencedReader(self.environ['wsgi.input'],
content_length,
self.max_entry_size)
content_type = self.environ.get('CONTENT_TYPE')
if content_type == 'application/json':
meth = self._lookup(method)
body = reader.read_chunk(sys.maxint)
return meth(args, body)
elif content_type.startswith('application/x-soledad-sync'):
# read one line and validate it
body_getline = reader.getline
if body_getline().strip() != '[':
raise http_app.BadRequest()
line = body_getline()
line, comma = utils.check_and_strip_comma(line.strip())
meth_args = self._lookup('%s_args' % method)
meth_args(args, line)
# handle incoming documents
if content_type == 'application/x-soledad-sync-put':
meth_put = self._lookup('%s_put' % method)
meth_end = self._lookup('%s_end' % method)
while True:
line = body_getline()
entry = line.strip()
if entry == ']': # end of incoming document stream
break
if not entry or not comma: # empty or no prec comma
raise http_app.BadRequest
entry, comma = utils.check_and_strip_comma(entry)
meth_put({}, entry)
if comma or body_getline(): # extra comma or data
raise http_app.BadRequest
return meth_end()
# handle outgoing documents
elif content_type == 'application/x-soledad-sync-get':
line = body_getline()
entry = line.strip()
meth_get = self._lookup('%s_get' % method)
return meth_get({}, line)
else:
raise http_app.BadRequest()
else:
raise http_app.BadRequest()
def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None):
"""
Parse incoming synchronization stream and insert documents in the
local database.
If an incoming document's encryption scheme is equal to
EncryptionSchemes.SYMKEY, then this method will decrypt it with
Soledad's symmetric key.
:param data: The body of the HTTP response.
:type data: str
:param return_doc_cb: A callback to insert docs from target.
:type return_doc_cb: function
:param ensure_callback: A callback to ensure we have the correct
target_replica_uid, if it was just created.
:type ensure_callback: function
:raise BrokenSyncStream: If C{data} is malformed.
:return: A dictionary representing the first line of the response got
from remote replica.
:rtype: list of str
"""
parts = data.splitlines() # one at a time
if not parts or parts[0] != '[':
raise BrokenSyncStream
data = parts[1:-1]
comma = False
if data:
line, comma = utils.check_and_strip_comma(data[0])
res = json.loads(line)
if ensure_callback and 'replica_uid' in res:
ensure_callback(res['replica_uid'])
for entry in data[1:]:
if not comma: # missing in between comma
raise BrokenSyncStream
line, comma = utils.check_and_strip_comma(entry)
entry = json.loads(line)
#-------------------------------------------------------------
# symmetric decryption of document's contents
#-------------------------------------------------------------
# if arriving content was symmetrically encrypted, we decrypt
# it.
doc = SoledadDocument(
entry['id'], entry['rev'], entry['content'])
if doc.content and ENC_SCHEME_KEY in doc.content:
if doc.content[ENC_SCHEME_KEY] == \
EncryptionSchemes.SYMKEY:
doc.set_json(decrypt_doc(self._crypto, doc))
#-------------------------------------------------------------
# end of symmetric decryption
#-------------------------------------------------------------
return_doc_cb(doc, entry['gen'], entry['trans_id'])
if parts[-1] != ']':
try:
partdic = json.loads(parts[-1])
except ValueError:
pass
else:
if isinstance(partdic, dict):
self._error(partdic)
raise BrokenSyncStream
if not data or comma: # no entries or bad extra comma
raise BrokenSyncStream
return res