def __init__(self, room_version: RoomVersion, v1_api: bool): self._response = SendJoinResponse([], [], {}) self._room_version = room_version # The V1 API has the shape of `[200, {...}]`, which we handle by # prefixing with `item.*`. prefix = "item." if v1_api else "" self._coro_state = ijson.items_coro( _event_list_parser(room_version, self._response.state), prefix + "state.item", use_float=True, ) self._coro_auth = ijson.items_coro( _event_list_parser(room_version, self._response.auth_events), prefix + "auth_chain.item", use_float=True, ) # TODO Remove the unstable prefix when servers have updated. # # By re-using the same event dictionary this will cause the parsing of # org.matrix.msc3083.v2.event and event to stomp over each other. # Generally this should be fine. self._coro_unstable_event = ijson.kvitems_coro( _event_parser(self._response.event_dict), prefix + "org.matrix.msc3083.v2.event", use_float=True, ) self._coro_event = ijson.kvitems_coro( _event_parser(self._response.event_dict), prefix + "event", use_float=True, )
def __init__(self, room_version: RoomVersion, v1_api: bool): self._response = SendJoinResponse([], []) # The V1 API has the shape of `[200, {...}]`, which we handle by # prefixing with `item.*`. prefix = "item." if v1_api else "" self._coro_state = ijson.items_coro( _event_list_parser(room_version, self._response.state), prefix + "state.item", ) self._coro_auth = ijson.items_coro( _event_list_parser(room_version, self._response.auth_events), prefix + "auth_chain.item", )
def __init__(self, room_version: RoomVersion): self._response = StateRequestResponse([], []) self._room_version = room_version self._coros: List[Generator[None, bytes, None]] = [ ijson.items_coro( _event_list_parser(room_version, self._response.state), "pdus.item", use_float=True, ), ijson.items_coro( _event_list_parser(room_version, self._response.auth_events), "auth_chain.item", use_float=True, ), ]
def stream_report(report_url, user, password): # Force the format query param to be set to format=json # Split query params off url_breakdown = report_url.split('?') # Gather all params that are not format if len(url_breakdown) == 1: params = [] else: params = [ x for x in url_breakdown[1].split('&') if not x.startswith('format=') ] # Add the format param params.append('format=json') param_string = '&'.join(params) # Put the url back together corrected_url = url_breakdown[0] + '?' + param_string # Get the data with requests.get(corrected_url, auth=(user, password), stream=True) as resp: resp.raise_for_status() # Set up our search key report_entry_key = b'Report_Entry' search_prefix = report_entry_key.decode('utf-8') + '.item' # NB This creates a "push" style interface with the ijson iterable # parser This sendable_list will be populated with intermediate # values by the items_coro() when send() is called. The # sendable_list must then be purged of values before it can be # used again. We have an explicit check for whether we find the # 'Report_Entry' key because if we do not find it the parser # yields 0 records instead of failing and this allows us to know # if the schema is changed records = ijson.sendable_list() coro = ijson.items_coro(records, search_prefix) found_key = False for chunk in resp.iter_content(chunk_size=512): if report_entry_key in chunk: found_key = True coro.send(chunk) for rec in records: yield rec del records[:] if not found_key: raise Exception( "Did not see '{}' key in response. Report does not conform to expected schema, failing." .format(report_entry_key)) coro.close()
def __init__(self, room_version: RoomVersion, v1_api: bool): self._response = SendJoinResponse([], [], event_dict={}) self._room_version = room_version self._coros: List[Generator[None, bytes, None]] = [] # The V1 API has the shape of `[200, {...}]`, which we handle by # prefixing with `item.*`. prefix = "item." if v1_api else "" self._coros = [ ijson.items_coro( _event_list_parser(room_version, self._response.state), prefix + "state.item", use_float=True, ), ijson.items_coro( _event_list_parser(room_version, self._response.auth_events), prefix + "auth_chain.item", use_float=True, ), ijson.kvitems_coro( _event_parser(self._response.event_dict), prefix + "event", use_float=True, ), ] if not v1_api: self._coros.append( ijson.items_coro( _partial_state_parser(self._response), "org.matrix.msc3706.partial_state", use_float="True", ) ) self._coros.append( ijson.items_coro( _servers_in_room_parser(self._response), "org.matrix.msc3706.servers_in_room", use_float="True", ) )
def _read_json_list_stream(self, response, multiple_values=False): stream = response.iter_content() objects = ijson.sendable_list() coro = ijson.items_coro(objects, "item", use_float=True, multiple_values=False) for chunk in stream: coro.send(chunk) if len(objects): for o in objects: yield o del objects[:]
def __init__(self, room_version: RoomVersion, v1_api: bool): self._response = SendJoinResponse([], [], {}) self._room_version = room_version # The V1 API has the shape of `[200, {...}]`, which we handle by # prefixing with `item.*`. prefix = "item." if v1_api else "" self._coro_state = ijson.items_coro( _event_list_parser(room_version, self._response.state), prefix + "state.item", use_float=True, ) self._coro_auth = ijson.items_coro( _event_list_parser(room_version, self._response.auth_events), prefix + "auth_chain.item", use_float=True, ) self._coro_event = ijson.kvitems_coro( _event_parser(self._response.event_dict), prefix + "org.matrix.msc3083.v2.event", use_float=True, )