def _parse_sync_stream(self, data, return_doc_cb): parts = data.splitlines() # one at a time if not parts or parts[0] != '[': raise BrokenSyncStream data = parts[1:-1] if data: line, comma = utils.check_and_strip_comma(data[0]) res = simplejson.loads(line) for entry in data[1:]: if not comma: # missing in between comma raise BrokenSyncStream line, comma = utils.check_and_strip_comma(entry) entry = simplejson.loads(line) doc = Document(entry['id'], entry['rev'], entry['content']) return_doc_cb(doc, entry['gen'], entry['trans_id']) if parts[-1] != ']': try: partdic = simplejson.loads(parts[-1]) except ValueError: pass else: if isinstance(partdic, dict): self._error(partdic) raise BrokenSyncStream if comma: # bad extra comma raise BrokenSyncStream return res
def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None): parts = data.splitlines() # one at a time if not parts or parts[0] != '[': raise BrokenSyncStream data = parts[1:-1] comma = False if data: line, comma = utils.check_and_strip_comma(data[0]) res = json.loads(line) if ensure_callback and 'replica_uid' in res: ensure_callback(res['replica_uid']) for entry in data[1:]: if not comma: # missing in between comma raise BrokenSyncStream line, comma = utils.check_and_strip_comma(entry) entry = json.loads(line) doc = Document(entry['id'], entry['rev'], entry['content']) return_doc_cb(doc, entry['gen'], entry['trans_id']) if parts[-1] != ']': try: partdic = json.loads(parts[-1]) except ValueError: pass else: if isinstance(partdic, dict): self._error(partdic) raise BrokenSyncStream if not data or comma: # no entries or bad extra comma raise BrokenSyncStream return res
def __call__(self): args = urlparse.parse_qsl(self.environ['QUERY_STRING'], strict_parsing=False) try: args = dict( (k.decode('utf-8'), v.decode('utf-8')) for k, v in args) except ValueError: raise BadRequest() method = self.environ['REQUEST_METHOD'].lower() if method in ('get', 'delete'): meth = self._lookup(method) return meth(args, None) else: # we expect content-length > 0, reconsider if we move # to support chunked enconding try: content_length = int(self.environ['CONTENT_LENGTH']) except (ValueError, KeyError): raise BadRequest if content_length <= 0: raise BadRequest if content_length > self.max_request_size: raise BadRequest reader = _FencedReader(self.environ['wsgi.input'], content_length, self.max_entry_size) content_type = self.environ.get('CONTENT_TYPE', '') content_type = content_type.split(';', 1)[0].strip() if content_type == 'application/json': meth = self._lookup(method) body = reader.read_chunk(sys.maxint) return meth(args, body) elif content_type == 'application/x-u1db-sync-stream': meth_args = self._lookup('%s_args' % method) meth_entry = self._lookup('%s_stream_entry' % method) meth_end = self._lookup('%s_end' % method) body_getline = reader.getline if body_getline().strip() != '[': raise BadRequest() line = body_getline() line, comma = utils.check_and_strip_comma(line.strip()) meth_args(args, line) while True: line = body_getline() entry = line.strip() if entry == ']': break if not entry or not comma: # empty or no prec comma raise BadRequest entry, comma = utils.check_and_strip_comma(entry) meth_entry({}, entry) if comma or body_getline(): # extra comma or data raise BadRequest return meth_end() else: raise BadRequest()
def __call__(self): args = urlparse.parse_qsl(self.environ['QUERY_STRING'], strict_parsing=False) try: args = dict( (k.decode('utf-8'), v.decode('utf-8')) for k, v in args) except ValueError: raise BadRequest() method = self.environ['REQUEST_METHOD'].lower() if method in ('get', 'delete'): meth = self._lookup(method) return meth(args, None) else: # we expect content-length > 0, reconsider if we move # to support chunked enconding try: content_length = int(self.environ['CONTENT_LENGTH']) except (ValueError, KeyError): raise BadRequest if content_length <= 0: raise BadRequest if content_length > self.max_request_size: raise BadRequest reader = _FencedReader(self.environ['wsgi.input'], content_length, self.max_entry_size) content_type = self.environ.get('CONTENT_TYPE') if content_type == 'application/json': meth = self._lookup(method) body = reader.read_chunk(sys.maxint) return meth(args, body) elif content_type == 'application/x-u1db-sync-stream': meth_args = self._lookup('%s_args' % method) meth_entry = self._lookup('%s_stream_entry' % method) meth_end = self._lookup('%s_end' % method) body_getline = reader.getline if body_getline().strip() != '[': raise BadRequest() line = body_getline() line, comma = utils.check_and_strip_comma(line.strip()) meth_args(args, line) while True: line = body_getline() entry = line.strip() if entry == ']': break if not entry or not comma: # empty or no prec comma raise BadRequest entry, comma = utils.check_and_strip_comma(entry) meth_entry({}, entry) if comma or body_getline(): # extra comma or data raise BadRequest return meth_end() else: raise BadRequest()
def test_check_and_strip_comma(self): line, comma = utils.check_and_strip_comma("abc,") self.assertTrue(comma) self.assertEqual("abc", line) line, comma = utils.check_and_strip_comma("abc") self.assertFalse(comma) self.assertEqual("abc", line) line, comma = utils.check_and_strip_comma("") self.assertFalse(comma) self.assertEqual("", line)
def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None): """ Does the same as parent's method but ensures incoming content will be decrypted. """ parts = data.splitlines() # one at a time if not parts or parts[0] != '[': raise BrokenSyncStream data = parts[1:-1] comma = False if data: line, comma = utils.check_and_strip_comma(data[0]) res = json.loads(line) if ensure_callback and 'replica_uid' in res: ensure_callback(res['replica_uid']) for entry in data[1:]: if not comma: # missing in between comma raise BrokenSyncStream line, comma = utils.check_and_strip_comma(entry) entry = json.loads(line) # decrypt after receiving from server. if not self._soledad: raise NoSoledadInstance() enc_json = json.loads(entry['content'])['_encrypted_json'] if not self._soledad.is_encrypted_sym(enc_json): raise DocumentNotEncrypted( "Incoming document from sync is not encrypted.") doc = LeapDocument(entry['id'], entry['rev'], encrypted_json=entry['content'], soledad=self._soledad) return_doc_cb(doc, entry['gen'], entry['trans_id']) if parts[-1] != ']': try: partdic = json.loads(parts[-1]) except ValueError: pass else: if isinstance(partdic, dict): self._error(partdic) raise BrokenSyncStream if not data or comma: # no entries or bad extra comma raise BrokenSyncStream return res
def _parse_received_doc_response(self, response): """ Parse the response from the server containing the received document. :param response: The body and headers of the response. :type response: tuple(str, dict) :return: (new_gen, new_trans_id, number_of_changes, doc_id, rev, content, gen, trans_id) :rtype: tuple """ # decode incoming stream parts = response.splitlines() if not parts or parts[0] != '[' or parts[-1] != ']': raise errors.BrokenSyncStream data = parts[1:-1] # decode metadata try: line, comma = utils.check_and_strip_comma(data[0]) metadata = None except (IndexError): raise errors.BrokenSyncStream try: metadata = json.loads(line) new_generation = metadata['new_generation'] new_transaction_id = metadata['new_transaction_id'] number_of_changes = metadata['number_of_changes'] except (ValueError, KeyError): raise errors.BrokenSyncStream # make sure we have replica_uid from fresh new dbs if self._ensure_callback and 'replica_uid' in metadata: self._ensure_callback(metadata['replica_uid']) # parse incoming document info doc_id = None rev = None content = None gen = None trans_id = None if number_of_changes > 0: try: entry = json.loads(data[1]) doc_id = entry['id'] rev = entry['rev'] content = entry['content'] gen = entry['gen'] trans_id = entry['trans_id'] except (IndexError, KeyError): raise errors.BrokenSyncStream return new_generation, new_transaction_id, number_of_changes, \ doc_id, rev, content, gen, trans_id
def __call__(self): """ Call an HTTP method of a resource. This method was rewritten to allow for a sync flow which uses one POST request for each transferred document (back and forth). Usual U1DB sync process transfers all documents from client to server and back in only one POST request. This is inconvenient for some reasons, as lack of possibility of gracefully interrupting the sync process, and possible timeouts for when dealing with large documents that have to be retrieved and encrypted/decrypted. Because of those, we split the sync process into many POST requests. """ args = urlparse.parse_qsl(self.environ['QUERY_STRING'], strict_parsing=False) try: args = dict( (k.decode('utf-8'), v.decode('utf-8')) for k, v in args) except ValueError: raise http_app.BadRequest() method = self.environ['REQUEST_METHOD'].lower() if method in ('get', 'delete'): meth = self._lookup(method) return meth(args, None) else: # we expect content-length > 0, reconsider if we move # to support chunked enconding try: content_length = int(self.environ['CONTENT_LENGTH']) except (ValueError, KeyError): raise http_app.BadRequest if content_length <= 0: raise http_app.BadRequest if content_length > self.max_request_size: raise http_app.BadRequest reader = http_app._FencedReader( self.environ['wsgi.input'], content_length, self.max_entry_size) content_type = self.environ.get('CONTENT_TYPE') if content_type == 'application/json': meth = self._lookup(method) body = reader.read_chunk(sys.maxint) return meth(args, body) elif content_type.startswith('application/x-soledad-sync'): # read one line and validate it body_getline = reader.getline if body_getline().strip() != '[': raise http_app.BadRequest() line = body_getline() line, comma = utils.check_and_strip_comma(line.strip()) meth_args = self._lookup('%s_args' % method) meth_args(args, line) # handle incoming documents if content_type == 'application/x-soledad-sync-put': meth_put = self._lookup('%s_put' % method) meth_end = self._lookup('%s_end' % method) entries = [] while True: line = body_getline() entry = line.strip() if entry == ']': # end of incoming document stream break if not entry or not comma: # empty or no prec comma raise http_app.BadRequest entry, comma = utils.check_and_strip_comma(entry) entries.append(entry) if comma or body_getline(): # extra comma or data raise http_app.BadRequest for entry in entries: meth_put({}, entry) return meth_end() # handle outgoing documents elif content_type == 'application/x-soledad-sync-get': line = body_getline() entry = line.strip() meth_get = self._lookup('%s_get' % method) return meth_get({}, line) else: raise http_app.BadRequest() else: raise http_app.BadRequest()
def __call__(self): """ Call an HTTP method of a resource. This method was rewritten to allow for a sync flow which uses one POST request for each transferred document (back and forth). Usual U1DB sync process transfers all documents from client to server and back in only one POST request. This is inconvenient for some reasons, as lack of possibility of gracefully interrupting the sync process, and possible timeouts for when dealing with large documents that have to be retrieved and encrypted/decrypted. Because of those, we split the sync process into many POST requests. """ args = urlparse.parse_qsl(self.environ['QUERY_STRING'], strict_parsing=False) try: args = dict( (k.decode('utf-8'), v.decode('utf-8')) for k, v in args) except ValueError: raise http_app.BadRequest() method = self.environ['REQUEST_METHOD'].lower() if method in ('get', 'delete'): meth = self._lookup(method) return meth(args, None) else: # we expect content-length > 0, reconsider if we move # to support chunked enconding try: content_length = int(self.environ['CONTENT_LENGTH']) except (ValueError, KeyError): raise http_app.BadRequest if content_length <= 0: raise http_app.BadRequest if content_length > self.max_request_size: raise http_app.BadRequest reader = http_app._FencedReader(self.environ['wsgi.input'], content_length, self.max_entry_size) content_type = self.environ.get('CONTENT_TYPE') if content_type == 'application/json': meth = self._lookup(method) body = reader.read_chunk(sys.maxint) return meth(args, body) elif content_type.startswith('application/x-soledad-sync'): # read one line and validate it body_getline = reader.getline if body_getline().strip() != '[': raise http_app.BadRequest() line = body_getline() line, comma = utils.check_and_strip_comma(line.strip()) meth_args = self._lookup('%s_args' % method) meth_args(args, line) # handle incoming documents if content_type == 'application/x-soledad-sync-put': meth_put = self._lookup('%s_put' % method) meth_end = self._lookup('%s_end' % method) while True: line = body_getline() entry = line.strip() if entry == ']': # end of incoming document stream break if not entry or not comma: # empty or no prec comma raise http_app.BadRequest entry, comma = utils.check_and_strip_comma(entry) meth_put({}, entry) if comma or body_getline(): # extra comma or data raise http_app.BadRequest return meth_end() # handle outgoing documents elif content_type == 'application/x-soledad-sync-get': line = body_getline() entry = line.strip() meth_get = self._lookup('%s_get' % method) return meth_get({}, line) else: raise http_app.BadRequest() else: raise http_app.BadRequest()
def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None): """ Parse incoming synchronization stream and insert documents in the local database. If an incoming document's encryption scheme is equal to EncryptionSchemes.SYMKEY, then this method will decrypt it with Soledad's symmetric key. :param data: The body of the HTTP response. :type data: str :param return_doc_cb: A callback to insert docs from target. :type return_doc_cb: function :param ensure_callback: A callback to ensure we have the correct target_replica_uid, if it was just created. :type ensure_callback: function :raise BrokenSyncStream: If C{data} is malformed. :return: A dictionary representing the first line of the response got from remote replica. :rtype: list of str """ parts = data.splitlines() # one at a time if not parts or parts[0] != '[': raise BrokenSyncStream data = parts[1:-1] comma = False if data: line, comma = utils.check_and_strip_comma(data[0]) res = json.loads(line) if ensure_callback and 'replica_uid' in res: ensure_callback(res['replica_uid']) for entry in data[1:]: if not comma: # missing in between comma raise BrokenSyncStream line, comma = utils.check_and_strip_comma(entry) entry = json.loads(line) #------------------------------------------------------------- # symmetric decryption of document's contents #------------------------------------------------------------- # if arriving content was symmetrically encrypted, we decrypt # it. doc = SoledadDocument( entry['id'], entry['rev'], entry['content']) if doc.content and ENC_SCHEME_KEY in doc.content: if doc.content[ENC_SCHEME_KEY] == \ EncryptionSchemes.SYMKEY: doc.set_json(decrypt_doc(self._crypto, doc)) #------------------------------------------------------------- # end of symmetric decryption #------------------------------------------------------------- return_doc_cb(doc, entry['gen'], entry['trans_id']) if parts[-1] != ']': try: partdic = json.loads(parts[-1]) except ValueError: pass else: if isinstance(partdic, dict): self._error(partdic) raise BrokenSyncStream if not data or comma: # no entries or bad extra comma raise BrokenSyncStream return res