def validate(self, frag): """ Validate input as correct JSON. If there are multiple JSON entities separated by a space, evaluate them separately. If there is an incomplete JSON array or object, buffer the fragment and wait to see if the object or array completes. Return a list of the valid JSON fragments passed in, or False. """ valid_input = [] jsonObj = [] end = 0 jsonString = "" try: decoder = JSONDecoder() fragLength = len(frag) while end != fragLength: obj, end = decoder.raw_decode(frag, idx=end) jsonObj.append(obj) valid_input.append(jsonObj) return valid_input except ValueError: print "This JSON is not valid. Shutting down now" sys.exit()
def parseJson(s, conn): """ Parses the json file and calls populate method for inserting the data into the 'repository' table input: s- json string conn- database connection """ try: _w=WHITESPACE.match decoder = JSONDecoder() s_len = len(s) end = 0 while end != s_len: obj, end = decoder.raw_decode(s, idx=_w(s, end).end()) end = _w(s, end).end() refType ='' if obj['type'] == 'CreateEvent': try: refType = obj['payload']['ref_type'] except Exception as e: print e if refType == 'repository': populateRepoTable(obj, conn) except Exception as e: #print 'Error in line:'+str(sys.exc_traceback.tb_lineno) print sys.exc_traceback.tb_lineno pass
def _read_message(self): """ Receives a message from the server or raises an RPCError. Returns: A header list and a json string payload. """ decoder = JSONDecoder() text_buffer = "" while True: try: read_chunk = self.socket.recv(8192) except socket.error: read_chunk = None if not read_chunk: raise RPCError("Connection Lost") cmd_start = 0 cmd_end = read_chunk.find("\00") while cmd_end != -1: output = text_buffer + read_chunk[cmd_start:cmd_end] text_buffer = "" try: header, payload_index = decoder.raw_decode(output, 0) if len(header) < 1 or len(header) > 2: raise RPCError("Received invalid message header from server") yield header, output[payload_index:] except ValueError: raise RPCError("Received invalid message from server") cmd_start = cmd_end + 1 cmd_end = read_chunk.find("\00", cmd_start) text_buffer += read_chunk[cmd_start:]
def parse_sls(self): """ Parse a YARN SLS trace file. This is a JSON file containing multiple job objects. """ json_decoder = JSONDecoder() job_objects = [] value_error_pattern = re.compile('Expecting .+ \(char (\d+)\)$') with open(self.sls_file) as sls_file: object_chunk = '' last_error_idx = -1 # Read file in chunks of lines. for chunk in lines_per_n(sls_file, LINES_TO_READ): # Remove all whitespace chunk = chunk.replace(" ", "") chunk = chunk.replace("\n", "") # Add (hopefully good) whitespace chunk = re.sub(r"{", r'{\n', chunk) chunk = re.sub(r"}", r'}\n', chunk) chunk = re.sub(r"\[", r'[\n', chunk) chunk = re.sub(r"\]", r']\n', chunk) # Further sanitize some JSON stuff chunk = re.sub(r"{\s*'?(\w)", r'{"\1', chunk) chunk = re.sub(r",\s*'?(\w)", r',"\1', chunk) chunk = re.sub(r"(\w)'?\s*:", r'\1":', chunk) chunk = re.sub(r":\s*'(\w+)'\s*([,}])", r':"\1"\2', chunk) object_chunk += chunk # Try to parse chunk read so far. chunk_parsing_done = False # Chunk may contain more than one object. while not chunk_parsing_done: try: parse_result = json_decoder.raw_decode(object_chunk) last_error_idx = -1 except ValueError as e: m = value_error_pattern.match(e.message) if m: # Get the index that the parsing error occurred on. idx = int(m.group(1)) if last_error_idx == -1 or last_error_idx != idx: # Chunk is not yet complete, keep reading. last_error_idx = idx break # The error at the current index was not due to an incomplete chunk. SLSParser._print_chunk(object_chunk) raise e # Add decoded job object to array job_objects.append(parse_result[0]) # Check if there's trailing data from another object object_end = parse_result[1] if object_end != len(object_chunk): # Trim chunk for the next object object_chunk = object_chunk[object_end + 1:] if not object_chunk.isspace(): chunk_parsing_done = True return job_objects
def loads_invalid_obj_list(s): decoder = JSONDecoder() s_len = len(s) objs = [] end = 0 while end != s_len: obj, end = decoder.raw_decode(s, idx=end) objs.append(obj) return objs
def loads_invalid_obj_list(s): decoder = JSONDecoder() s_len = len(s) objs = [] end = 0 while end != s_len: obj, end = decoder.raw_decode(s, idx=end) objs.append(obj) #if obj['repository'] is not None: etlMySQLWatch(obj)
def json_objects(input): decoder = JSONDecoder() next_obj_pat = re.compile("\S") idx = 0 while True: obj, stop = decoder.raw_decode(contents[idx:]) yield (obj, idx, idx+stop) m = next_obj_pat.search(contents, idx + stop) if m: idx = m.start() else: break
def read(self, bs: Union[bytes, bytearray], columns: Dict[str, Type], options: Options) -> Iterator[JsonValue]: s = bs.decode('utf8') decoder = JSONDecoder() offset = 0 try: while True: obj, offset = decoder.raw_decode(s, offset) offset += 1 # advances past the newline delimiter yield obj except JSONDecodeError: # finally arrived at the end of the string (or actually got malformed json from s3, supposed not to) pass
def loads_invalid_obj_list(filepass): s = open(filepass, 'r').readline() #s=filepass.readline() # s=unicode_string.encode(s) decoder = JSONDecoder() s_len = len(s) print(len(s)) objs = [] end = 0 while end != s_len: obj, end = decoder.raw_decode(s, idx=end) objs.append(obj) #print (decoder.raw_decode(s,indx=end)) return objs
def loads_invalid_obj_list(s): try: decoder = JSONDecoder() s_len = len(s) end = 0 while end != s_len: obj, end = decoder.raw_decode(s, idx=end) if obj['type'] =='PushEvent' or obj['type'] =='WatchEvent' or obj['type'] =='FollowEvent': etlMySQL(obj) except Exception as e: print 'Error in line:'+str(sys.exc_traceback.tb_lineno) print e pass
def decode(s): try: _w=WHITESPACE.match decoder = JSONDecoder() s_len = len(s) end = 0 while end != s_len: obj, end = decoder.raw_decode(s, idx=_w(s, end).end()) end = _w(s, end).end() if obj['type'] =='PushEvent' or obj['type'] =='WatchEvent' or obj['type'] =='FollowEvent': print obj['type'] etlMySQL(obj) except Exception as e: print 'Error in line:'+str(sys.exc_traceback.tb_lineno) print e pass
async def read_json( socket: trio.socket.SocketType, buffer: io.StringIO, decoder: json.JSONDecoder = decoder, ) -> JSON: request: JSON while True: data = await socket.recv(1024) buffer.write(data.decode()) bad_prefix, raw_request = strip_non_json_prefix(buffer.getvalue()) if bad_prefix: logger.info("Client started request with non json data: %r", bad_prefix) await write_error(socket, f"Cannot parse json: {bad_prefix}") raise DecodingError(f"Invalid JSON payload: prefix={bad_prefix}") if len(raw_request) > MAXIMUM_RPC_PAYLOAD_SIZE: error_msg = ( f"RPC payload exceeds maximum size: {len(raw_request)} " f"> {MAXIMUM_RPC_PAYLOAD_SIZE}") await write_error(socket, error_msg) raise DecodingError(error_msg) try: request, offset = decoder.raw_decode(raw_request) except json.JSONDecodeError: # invalid json request, keep reading data until a valid json is formed if raw_request: logger.debug( "Invalid JSON, waiting for rest of message: %r", raw_request, ) else: await trio.sleep(0.01) continue buffer.seek(0) buffer.write(raw_request[offset:]) buffer.truncate() break return request
def imgurl_from_id(raw, tbnid): from json import JSONDecoder q = '"{}",['.format(tbnid) start_pos = raw.index(q) if start_pos < 100: return jd = JSONDecoder() data = jd.raw_decode('[' + raw[start_pos:])[0] # from pprint import pprint # pprint(data) url_num = 0 for x in data: if isinstance(x, list) and len(x) == 3: q = x[0] if hasattr(q, 'lower') and q.lower().startswith('http'): url_num += 1 if url_num > 1: return q
def plot_file(self, file): print("plotting " + file, flush=True) if "dummy" in file: return self.file = self.open(file, "r") self.filename = file.split("/")[-1].split(".json")[0] self.fullfilename = file lines = self.file.read() while (not lines or len(lines) < 4) and self.seconds > 0: time.sleep(self.seconds) lines = self.file.read() if isinstance(lines, bytes): lines = lines.decode("utf-8") try: progress = json.loads(lines) # valid file except Exception as e: # print(e) # # print("-------------\n") # print("\n\n" + lines + "\n\n") try: progress = json.loads(lines[:-3] + "]") # unclosed list print("unclosed list json decoding...", flush=True) except Exception as e: # this is some bug in json.decode (like really, there is some stateful leak that makes this behave differently when called twice) print(e) raw = JSONDecoder( object_hook=None, object_pairs_hook=None ) # solution is to just skip the check for length in the decoding which seems erroneous progress = raw.raw_decode(lines[:-3] + "]") print("raw json decoding...", flush=True) print(progress, flush=True) extracted = {} for field in self.filter.fields: extracted[tuple(field)] = self.filter.extract(field, progress) print(extracted, flush=True) self.plot(extracted) plt.show() print("Initial plot should be displayed....") return extracted
def parse_sls(self): """ Parse a YARN SLS trace file. This is a JSON file containing multiple job objects. """ json_decoder = JSONDecoder() job_objects = [] with open(self.sls_file) as sls_file: object_chunk = '' # Read file in chunks of lines. for chunk in lines_per_n(sls_file, LINES_TO_READ): # Remove all whitespace chunk = chunk.replace(" ", "") chunk = chunk.replace("\n", "") # Add (hopefully good) whitespace chunk = re.sub(r"{", r'{\n', chunk) chunk = re.sub(r"}", r'}\n', chunk) chunk = re.sub(r"\[", r'[\n', chunk) chunk = re.sub(r"\]", r']\n', chunk) # Further sanitize some JSON stuff chunk = re.sub(r"{\s*'?(\w)", r'{"\1', chunk) chunk = re.sub(r",\s*'?(\w)", r',"\1', chunk) chunk = re.sub(r"(\w)'?\s*:", r'\1":', chunk) chunk = re.sub(r":\s*'(\w+)'\s*([,}])", r':"\1"\2', chunk) object_chunk += chunk # Try to parse chunk read so far. chunk_parsing_done = False # Chunk may contain more than one object. while not chunk_parsing_done: try: parse_result = json_decoder.raw_decode(object_chunk) except ValueError: # Chunk is not yet complete, keep reading. break # Add decoded job object to array job_objects.append(parse_result[0]) # Check if there's trailing data from another object object_end = parse_result[1] if object_end != len(object_chunk): # Trim chunk for the next object object_chunk = object_chunk[object_end + 1:] else: chunk_parsing_done = True return job_objects
def parse_message(self, message): tag, _, data = message.partition(" ") if tag in CLIENT: parsed = (CLIENT[tag],) elif tag in SERVER: parsed = (SERVER[tag],) else: raise ValueError("Unknown message type %s" % tag) decoder = JSONDecoder() while data: data = data.lstrip() try: decoded, end = decoder.raw_decode(data) except JSONDecodeError: break else: parsed += (decoded,) data = data[end:] return parsed
def parse_line(cls, line): role = "" tag, data = splart(line.strip()) fields = [] if tag.endswith(":"): role = tag.rstrip(":") tag, data = splart(data) decoder = JSONDecoder() while data: data = data.lstrip() try: decoded, end = decoder.raw_decode(data) except ValueError: fields.append(data) data = "" else: fields.append(decoded) data = data[end:] return role, tag, fields
def parse_message(self, message): tag, _, data = message.partition(" ") if tag in CLIENT: parsed = (CLIENT[tag], ) elif tag in SERVER: parsed = (SERVER[tag], ) else: raise ValueError("Unknown message type %s" % tag) decoder = JSONDecoder() while data: data = data.lstrip() try: decoded, end = decoder.raw_decode(data) except JSONDecodeError: break else: parsed += (decoded, ) data = data[end:] return parsed
def parse_topo(self): """ Parse a YARN SLS topology file. This is a JSON file containing multiple rack configurations. """ json_decoder = JSONDecoder() rack_objects = [] with open(self.topo_file) as topo_file: lines = "".join(topo_file.readlines()).strip() done_parsing_file = False while not done_parsing_file: try: rack_object, object_end = json_decoder.raw_decode(lines) except ValueError as e: LOG.exception("Unable to parse topology file", exc_info=e) break rack_objects.append(rack_object) if object_end != len(lines): lines = lines[object_end + 1:] else: done_parsing_file = True return rack_objects
def extract_json_objects( text: str, decoder: JSONDecoder = JSONDecoder() ) -> Iterator[dict[str, int | dict[Any, Any]]]: """Find JSON objects in text, and yield the decoded JSON data Does not attempt to look for JSON arrays, text, or other JSON types outside of a parent JSON object. """ pos = 0 while True: match = text.find("{", pos) if match == -1: break try: result, index = decoder.raw_decode(text[match:]) yield result pos = match + index except ValueError: pos = match + 1
def eventsListBuildFromList(filename): with open(filename, 'r') as myfile: s = myfile.read().replace('\n', '') decoder = JSONDecoder() s_len = len(s) Events = [] end = 0 while end != s_len: Event, end = decoder.raw_decode(s, idx=end) Events.append(Event) data = [] for e in Events: data.append(pandas.DataFrame.from_dict(e, orient='index')) Events = pandas.concat(data) for it in range(Events['attribute_count'].size): if Events['attribute_count'][it] is None: Events['attribute_count'][it] = '0' else: Events['attribute_count'][it] = int(Events['attribute_count'][it]) Events = Events.set_index('id') return Events
def parse_message(self, message): tag, _, data = message.partition(" ") v = self.bolt_version if tag in CLIENT[v]: parsed_tag = CLIENT[v][tag] elif tag in SERVER[v]: parsed_tag = SERVER[v][tag] else: raise ValueError("Unknown message type %s" % tag) decoder = JSONDecoder() parsed = [] while data: data = data.lstrip() try: decoded, end = decoder.raw_decode(data) except JSONDecodeError: break else: parsed.append(decoded) data = data[end:] return Structure(parsed_tag, *parsed)
def eventsListBuildFromList(filename): with open(filename, "r") as myfile: s = myfile.read().replace("\n", "") decoder = JSONDecoder() s_len = len(s) Events = [] end = 0 while end != s_len: Event, end = decoder.raw_decode(s, idx=end) Events.append(Event) data = [] for e in Events: data.append(pandas.DataFrame.from_dict(e, orient="index")) Events = pandas.concat(data) for it in range(Events["attribute_count"].size): if Events["attribute_count"][it] is None: Events["attribute_count"][it] = "0" else: Events["attribute_count"][it] = int(Events["attribute_count"][it]) Events = Events.set_index("id") return Events
def _hosts(self): if not exists(self.hostlist_path): logger.warning('{} not found, no hosts configured', self.hostlist_path) return pos = 0 decoder = JSONDecoder() with open(self.hostlist_path) as file: document = file.read() while True: match = NOT_WHITESPACE.search(document, pos) if not match: break pos = match.start() obj, pos = decoder.raw_decode(document, pos) if 'hosts' in obj.keys(): for host in obj['hosts']: yield Host(hostname=host[1], port=host[2], username=host[3], password=host[4])
def parse_message(self, message): tag, _, data = message.partition(" ") v = self.bolt_version if tag in CLIENT[v]: parsed_tag = CLIENT[v][tag] elif tag in SERVER[v]: parsed_tag = SERVER[v][tag] else: raise ValueError("Unknown message type %s" % tag) decoder = JSONDecoder() parsed = [] while data: data = data.lstrip() try: decoded, end = decoder.raw_decode(data) except ValueError: break else: parsed.append(decoded) data = data[end:] return Structure(parsed_tag, *parsed)
def eventsListBuildFromList(filename): with open('testt', 'r') as myfile: s=myfile.read().replace('\n', '') decoder = JSONDecoder() s_len = len(s) Events = [] end = 0 while end != s_len: Event, end = decoder.raw_decode(s, idx=end) Events.append(Event) data = [] for e in Events: data.append(pd.DataFrame.from_dict(e, orient='index')) Events = pd.concat(data) for it in range(Events['attribute_count'].size): if Events['attribute_count'][it] == None: Events['attribute_count'][it]='0' else: Events['attribute_count'][it]=int(Events['attribute_count'][it]) Events = Events.set_index('id') return Events
def scrap_page(self, response): """ this function is called after the getting response from the request created in the parse fn. scrapped data is stored in the directory call scrapedData in the directory myntra/spiders """ script = response.xpath( '/html/body/script[contains(text(),"searchData")]/text()' ).extract_first() script = script.replace("window.__myx = ", "") script = self.turn_to_json(script) decoder = JSONDecoder() script = decoder.raw_decode(script)[0] f = open( os.path.join(datafolder, response.url.replace('/', '-') + ".csv"), 'w') products_list = script["searchData"]["results"]["products"] fieldnames = products_list[0].keys() dictwriter = DictWriter(f, fieldnames=fieldnames) dictwriter.writeheader() for product in products_list: dictwriter.writerow(product) f.close()
def send_raw_command(self, name, *args): """ Send a command providing a structured list of command inputs but no Schema. Args: *args <various types>: The command's inputs as python objects Returns: A tuple containing a Result object and possibly other native python objects resulting from a vanilla JSON parse of the function's outputs. Raises: RPCError if it was not possible to obtain a result from the host application. """ try: result, reply = self.send_json_command(name, SchemaServices.write(args)) if result.is_rpc_error(): raise RPCError(str(result)) decoder = JSONDecoder() reply_tuple = (result,) + decoder.raw_decode(reply) return reply_tuple if len(reply_tuple) != 1 else reply_tuple[0] except ValueError as e: raise RPCError(str(e))
def get_messages( decoder: json.JSONDecoder, buf: str) -> Iterator[Tuple[str, Any]]: """Parse individual messages from an input buffer. :decoder: A json decoder. :buf: The buffer to decode. :yield: Tuples of (unused-buf, message). """ message = 'dummy' while buf and message: message = '' try: message, index = decoder.raw_decode(buf) except json.JSONDecodeError: log.info(f'Decode error: {buf!r}') return except Exception: f = io.StringIO() traceback.print_exc(file=f) log.info(f.getvalue()) raise buf = buf[index:].lstrip() yield buf, message
class Client(object): """ Client for connecting to a Vicon application (for example Shogun Live) using the TerminalServer protocol. Currently this is a simple implementation that just supports callbacks and blocking functions. Connection to the server is performed synchronously when the client is created. Note that callbacks are called from within the client's receive thread and therefore it is not permissible to call any functions of the client from within callbacks. If the client was not successful in connecting to the host application, any attempt to send commands will raise RPCError: NotConnected. """ check_schemas_schema = Schema.make_function("Terminal.CheckSchemas", Schema.make_list(Schema(Schema.Type.EString))) check_schemas_schema.add_input("Schemas", Schema.make_list(Schema.make_ref("Schema"))) SchemaServices.register_schema(None, check_schemas_schema) def __init__(self, host="localhost", port=52800, connect_timeout_seconds=10, send_timeout_seconds=1, client_failed_callback=None): """ Initialise with the hostname (or IP address) and port at which the server may be reached. Args: host <string>: Name or ip address for the server port <int>: Server port to connect to. client_failed_callback < python function taking a string > : optionally supply a function to be called when the client stops or fails """ self.connect_timeout_seconds = connect_timeout_seconds self.send_timeout_seconds = send_timeout_seconds self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.setblocking(send_timeout_seconds) self.server_endpoint = (host, port) self.version = None self.client_failed_callback = client_failed_callback self.message_id_generator = 0 self.callback_id_generator = 0 self.pending_messages = {} self.callback_map = {} self.thread = None self.condition = Condition(Lock()) self.connected = False self.decoder = JSONDecoder() self._message_generator = None self._connect() def stop(self): """ Stop the client and close its connection. """ with self.condition: try: self.socket.shutdown(socket.SHUT_RDWR) self.socket.close() except socket.error: pass self.connected = False thread = self.thread if thread: thread.join() def server_version(self): """ Get the protocol version of the server. Returns: A tuple (MajorVersionNumber, MinorVersionNumber) or None if unknown. """ return self.version def check_schemas(self, schema_list): """ Get the server to confirm if it supports the provided schemas. returns: A list of schema type_names for any of the schemas that are NOT supported""" return self.send_command("Terminal.CheckSchemas", schema_list) def send_json_command(self, name, json_arg_string="[]"): """ Send a command, providing the arguments as a single JSON string. Args: name <string>: The remote command name. json_arg_string <string>: The command arguments as a JSON string (should be a valid JSON list) Returns: A result code and a JSON string. If the result code does not represent an RPC failure then the JSON string will contain a valid JSON list containing any additional outputs of the command. Raises: RPCError if it was not possible to obtain a result from the host application. """ with self.condition: return self._locked_send_command(name, json_arg_string) def send_raw_command(self, name, *args): """ Send a command providing a structured list of command inputs but no Schema. Args: *args <various types>: The command's inputs as python objects Returns: A tuple containing a Result object and possibly other native python objects resulting from a vanilla JSON parse of the function's outputs. Raises: RPCError if it was not possible to obtain a result from the host application. """ try: result, reply = self.send_json_command(name, SchemaServices.write(args)) if result.is_rpc_error(): raise RPCError(str(result)) decoder = JSONDecoder() reply_tuple = (result,) + decoder.raw_decode(reply) return reply_tuple if len(reply_tuple) != 1 else reply_tuple[0] except ValueError as e: raise RPCError(str(e)) def send_command(self, name, *args): """ Send a command by providing the command's name and a structured list of command inputs. This function requires that the command's schema has been registered with SchemaServices. If the function fails due to RPC mechanics the function will raise an RPCError. Args: schema < Schema >: The command's schema (see schema.Schema). *args <various types>: The command's inputs as python objects Returns: A tuple of command outputs. The first element of this tuple is the command's return value if any. Raises: RPCError if it was not possible to obtain a result from the host application. """ with self.condition: if not self.connected: raise RPCError(str(Result.RPCNotConnected)) schema = SchemaServices.schema(name) if schema is None: raise RPCError(name + "Schema is not registered with SchemaServices") try: result, reply = self._locked_send_command(name, SchemaServices.write(args)) if result.is_rpc_error(): raise RPCError(str(result)) if schema.sub_schemas and schema.sub_schemas[0][1].role == Schema.Role.EResult: reply_tuple = (result,) + SchemaServices.read(reply, schema) else: reply_tuple = SchemaServices.read(reply, schema) return reply_tuple if len(reply_tuple) != 1 else reply_tuple[0] except ValueError as e: raise RPCError(str(e)) def add_schema_callback(self, callback_name, function, schema): """ Add a callback, specifying the callback's schema if any. Warning: Callbacks are called directly from within the client's receive thread. Therefore it is not safe to call client commands from within callback functions. Args: callback_name < string >: The name of the callback. function < python function >: The function to call when the callback is invoked. The function should take arguments that correspond to the callback's schema. schema < Schema >: The callback's schema or 'None'. If 'None' then the 'function' should take only a single argument that is the JSON string of the callback arguments. Note that add_schema_callback() does not attempt validate the callback's schema with the server. Use check_schemas() for this. Incorrect schemas will result in the callback function not being invoked. Returns: A result code and a callback id that can be used to remove the callback. """ result = Result.Ok callback_id = 0 with self.condition: self.callback_id_generator += 1 if callback_name not in self.callback_map: result, _reply = self._locked_send_command("Terminal.EnableCallback", "[\"" + callback_name + "\",true]") if result: callback_id = self.callback_id_generator self.callback_map[callback_name] = [(callback_id, schema, function)] else: callback_id = self.callback_id_generator self.callback_map[callback_name].append((callback_id, schema, function)) return result, (callback_name, callback_id) def add_callback(self, callback_name, function): """ Add a callback. The callback's schema must have deen registered with SchemaServices. Warning: Callbacks are called directly from within the client's receive thread. Therefore it is not safe to call client commands from within callback functions. Args: callback_name < string >: The name of the callback (whose schema has been registered with SchemaServices. Note that add_schema_callback() does not attempt validate the callback's schema with the server. Use check_schemas() for this. Incorrect schemas will result in the callback function not being invoked. function < python function >: The function to call when the callback is invoked. The function should take arguments that correspond to the callback's schema. Returns: A result code, and a callback id that can be used to remove the callback or None if the callback could not be added. """ schema = SchemaServices.schema(callback_name) if schema is None: return Result.NotFound, None return self.add_schema_callback(callback_name, function, schema) def remove_callback(self, callback_id): """ Remove a callback. Args: callback_id < int >: A non-zero callback Id returned from a successfull call to add_callback(). Returns: A result code that indicated success if the callback was removed """ with self.condition: callback_entries = self.callback_map.get(callback_id[0], None) if callback_entries is None: return Result.NotFound callback_entry = next((x for x in callback_entries if x[0] == callback_id[1]), None) if not callback_entry: return Result.NotFound callback_entries.remove(callback_entry) if not callback_entries: self._locked_send_command("Terminal.EnableCallback", "[\"" + callback_id[0] + "\",false]") self.callback_map.pop(callback_id[0]) return Result.Ok def __enter__(self): return self def __exit__(self, type, value, traceback): self.stop() def _connect(self): """ Called by__init__ to connect to server """ deadline_time = datetime.now() + timedelta(seconds=self.connect_timeout_seconds) if self.socket.connect_ex(self.server_endpoint) == 0: with self.condition: self.connected = True self._message_generator = self._read_message() self.thread = Thread() self.thread.run = self._read_loop self.thread.setDaemon(True) self.thread.start() while self.connected and (self.version is None) and datetime.now() < deadline_time: self.condition.wait(0.1) if not self.version: self.connected = False if self.client_failed_callback: self.client_failed_callback("Failed to receive server version") self.client_failed_callback = None elif self.client_failed_callback: self.client_failed_callback("Failed to connect to server") self.client_failed_callback = None def _locked_send_command(self, name, payload): """ Sends a command using a command name and a json string argument payload. """ if not self.connected: return Result.RPCNotConnected, "" deadline_time = datetime.now() + timedelta(seconds=self.send_timeout_seconds) self.message_id_generator += 1 message_id = self.message_id_generator message = "[\"" + name + "\"," + str(message_id) + "]" + payload + "\00" self.pending_messages[message_id] = (None, "") self.socket.send(message) while datetime.now() < deadline_time: pending_message = self.pending_messages.get(message_id, None) if pending_message is None: return Result.RPCFailed, "" if pending_message[0] is not None: self.pending_messages.pop(message_id) return pending_message self.condition.wait(0.1) return Result.TimedOut, "Send timeout out after {} seconds".format(self.send_timeout_seconds) def _read_message(self): """ Receives a message from the server or raises an RPCError. Returns: A header list and a json string payload. """ decoder = JSONDecoder() text_buffer = "" while True: try: read_chunk = self.socket.recv(8192) except socket.error: read_chunk = None if not read_chunk: raise RPCError("Connection Lost") cmd_start = 0 cmd_end = read_chunk.find("\00") while cmd_end != -1: output = text_buffer + read_chunk[cmd_start:cmd_end] text_buffer = "" try: header, payload_index = decoder.raw_decode(output, 0) if len(header) < 1 or len(header) > 2: raise RPCError("Received invalid message header from server") yield header, output[payload_index:] except ValueError: raise RPCError("Received invalid message from server") cmd_start = cmd_end + 1 cmd_end = read_chunk.find("\00", cmd_start) text_buffer += read_chunk[cmd_start:] def _read_server_version(self): """ Receives and validates the server protocol version. """ header, json_message = next(self._message_generator) with self.condition: if header is None or header[0] != "ViconTerminal": raise RPCError("Server endpoint was not a ViconTerminal") try: decoded, _index = self.decoder.raw_decode(json_message, 0) if len(decoded) == 2 and isinstance(decoded[0], int) and isinstance(decoded[1], int): self.version = (decoded[0], decoded[1]) self.condition.notify_all() return except ValueError: pass raise RPCError("Could not interpret ViconTerminal version from server") def _read_loop(self): """ Receives replies and callbacks from the Server until a failure is encountered. """ try: self._read_server_version() while True: header, json_message = next(self._message_generator) if len(header) == 2: # is command reply command_id = header[0] result = Result(header[1]) with self.condition: if command_id in self.pending_messages: if result.is_rpc_error(): self.pending_messages[command_id] = (result, None) else: self.pending_messages[command_id] = (result, json_message) self.condition.notify_all() elif len(header) == 1: # is callback with self.condition: callback_name = header[0] callback_entries = self.callback_map.get(callback_name, []) for callback_entry in callback_entries: # decode json callback if we have a schema if callback_entry[1] is not None: try: arg_tuple = SchemaServices.read(json_message, callback_entry[1]) callback_entry[2](*arg_tuple) except Exception as e: # if the user callback raises, we want to provide the stack trace so they can debug raise RPCError('User callback raised exception: ' + traceback.format_exc()) # otherwise call the callback with the json text else: callback_entry[2](json_message) except RPCError as e: # when connection fails we complete all pending messages with self.condition: self.pending_messages.clear() self.connected = False self.condition.notify_all() if self.client_failed_callback: self.client_failed_callback(str(e))
class SocketReadWrite(object): """ Object that will read and write from a socket. Used primarily as a base class for the Client and Server, or an instance for nodes. Logging is done against root logger. """ blocking_retry_max = 5 def __init__(self, **kwargs): self.verbose = kwargs.get('verbose', 0) peertuple = kwargs.get('peertuple', None) selectable = kwargs.get('selectable', True) sock = kwargs.get('sock', None) # Allow chars such as CRLF. A fresh instance seems to be important # in a reconnect. self.jsond = JSONDecoder(strict=False) if sock is None: self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) else: self._sock = sock self._created_blocking = not selectable self.safe_setblocking() if peertuple is None: self._host = '' self._port = 0 self._str = '' else: # A dead socket can't getpeername so cache it now self._host, self._port = peertuple self._str = '{0}:{1}'.format(*peertuple) self.clear() self.inOOB = [] self.outbytes = bytes() self.blocking_retry = 0 def __str__(self): return self._str def safe_setblocking(self, value=None): '''If None, restore to created value, else use specified value''' if self._sock is not None: if value is None: value = self._created_blocking self._sock.setblocking(bool(value)) def close(self): if self._sock is None: return try: self._sock.shutdown(socket.SHUT_RDWR) except Exception as e: # already closed, whatever pass self._sock.close() self._sock = None # Force AttributeError on methods self._str += ' (closed)' def fileno(self): '''Allows this object to be used in select''' return -1 if self._sock is None else self._sock.fileno() def check_blocking_retry_max(self): self.blocking_retry += 1 time.sleep(.2) return (self.blocking_retry > self.blocking_retry_max) def reset_blocking_retry(self): self.blocking_retry = 0 #---------------------------------------------------------------------- # Send stuff def send_all(self, obj, JSON=True): """ Send an object after optional transformation Args: obj: the object to be sent, None means work off backlog Returns: True or raised error. """ self.sent = 0 if JSON: # Error possible here: "not JSON serializable", let it raise outbytes = dumps(obj).encode() else: outbytes = obj.encode() logging.info( '%s: sending %s' % (self, 'NULL' if obj is None else '%d bytes' % len(outbytes))) # socket.sendall will do so and return None on success. If not, # an error is raised with no clue on byte count. Do it myself. self.outbytes += outbytes try: while len(self.outbytes): # This seems to throw errno 11 on its own, but just in case # do it myself. BTW, EAGAIN == EWOULDBLOCK n = self._sock.send(self.outbytes) if not n: raise OSError(errno.EWOULDBLOCK, 'full') self.sent += n self.outbytes = self.outbytes[n:] self.reset_blocking_retry() return True except BlockingIOError as e: # Far side is full. FIXME: raising OSError is a weak response if self.check_blocking_retry_max(): raise OSError(errno.EWOULDBLOCK, 'blocking_retry_max') except OSError as e: if e.errno == errno.EPIPE: msg = 'closed by client' elif e.errno != errno.EBADF: msg = 'closed earlier' msg = '%s: %s' % (self, msg) raise OSError(errno.ECONNABORTED, msg) except AttributeError as e: # During retry of a closed socket. FIXME: delay the close? raise OSError(errno.ECONNABORTED, 'Socket closed on prior error') except Exception as e: logging.error('%s: send_all failed: %s' % (self, str(e))) set_trace() pass raise return False def send_result(self, result, JSON=True): try: self.last_errmsg = '' return self.send_all(result, JSON) # True or raise except Exception as e: # could be blocking IO self.last_errmsg = '%s: %s' % (self, str(e)) logging.error(self.last_errmsg) return False #---------------------------------------------------------------------- # Receive stuff _bufsz = 8192 # max recv. FIXME: preallocate this? _bufhi = 2 * _bufsz # Not sure I'll keep this _OOBlimit = 20 # when to dump a flood def recv_all(self): """ Receive the next part of a message and decode it to a python3 string. Args: Returns: """ needmore = False # Should be entered with an empty buffer while True: if self.inOOB: # make caller deal with OOB first return None last = len(self.instr) if last: if last > 60: logging.debug('INSTR: %d bytes' % last) else: logging.debug('INSTR: %s' % self.instr) appended = 0 # First time through OR go-around with partial buffer? if not last or needmore: try: if last: # maybe trying to finish off a fragment self._sock.settimeout(0.5) # akin to blocking mode self.instr += self._sock.recv(self._bufsz).decode('utf-8') appended = len(self.instr) - last logging.info('%s: recvd %d bytes' % (self._str, appended)) if not appended: # Far side is gone without timeout msg = '%s: closed by remote' % str(self) self.close() # May lead to AttributeError below raise OSError(errno.ECONNABORTED, msg) except ConnectionAbortedError as e: # see two lines up raise except BlockingIOError as e: # Not ready; only happens on a fresh read with non-blocking # mode (ie, can't happen in timeout mode). Get back to # select, or just re-recv? set_trace() # probably created with selectable=True if not self._created_blocking: return None continue except socket.timeout as e: pass except Exception as e: # AttributeError is ops on closed socket. Other stuff # can just go through. if self._sock is None: raise OSError(errno.ECONNABORTED, str(self)) self.safe_setblocking() # just in case it's live raise self.safe_setblocking() # undo timeout (idempotent) # Only the JSON decode can be allowed to throw an error. # Json raw_decode() can return something other than a dict. # Scalars indicate a partial parsing of the middle of a full # JSON response. Let the parser keep at it until it exhausts # those symbols. while self.instr: try: if len(self.inOOB) > self._OOBlimit and \ len(self.instr) < self._bufhi: return None # Deal with this part of the flood result, nextjson = self.jsond.raw_decode(self.instr) self.instr = self.instr[nextjson:] if not isinstance(result, dict): continue OOBmsg = result.get('OOBmsg', False) if OOBmsg: # and that's the whole result self.inOOB.append(OOBmsg) continue return result except ValueError as e: # Bad JSON conversion. Since I'm using raw_decode # multiple messages are no longer a problem; just walk # through them with nextjson above. xattrs can go past # _bufsiz, or perhaps an OOB flood filled instr. Either # way I need more bytes by breaking back into recv loop. # Is this a failure after re-read and if so did it help? if needmore and needmore == self.instr: # A "return None" might be better here when OOB # processing is actually used. break # There's only a 1 in bufsz chance that a read really # ended exactly on the boundary. In other words, a # full read probably means there's more. if appended >= self._bufsz: # decode() might make more needmore = self.instr # the '+='rebinds instr break # Does it at least smell like JSON? Maybe it's the middle # of one message and the beginning of another. That's # authoritative, so remove pre-cruft and try again. leftright = self.instr.find('}{') if leftright != -1: self.instr = self.instr[leftright + 1:] continue # What about a good start, but unfinished end? Might # not be able to tell because of sub-objects. leftcurly = self.instr.find('{') if leftcurly == 0: needmore = self.instr break # at this stage, it's not recoverable self.clear() return None except Exception as e: raise RuntimeError( 'JSON decode results have been misinterpreted') raise RuntimeError( 'Unexpectedly reached end of JSON parsing loop') def clear(self): self.instr = '' def clearOOB(self): self.inOOB = []
def run(log_location, base_dir): cache = set([]) file_cache = {} expected_keys = {"i", "o", "d", "p", "t", "r", "w"} # log_location = 'biglog.log' # log_location = '/var/log/nginx/coronaviruscheck.org/postdata.log.1' with open(log_location) as json_file: decoder = JSONDecoder() batch_number = "/" + datetime.now().strftime( '%Y-%m-%dT%H:%M:%S') + '_' + token_hex(15) + ".csv" for line in json_file: if line != "\n": try: log_data = decoder.raw_decode(line) except: continue bulk_write = {} if 'z' in log_data[0]: for interaction in log_data[0]['z']: event_date = datetime.fromtimestamp( int(interaction["w"])) base_path = [ base_dir, event_date.year, str(event_date.month).zfill(2), str(event_date.day).zfill(2), ] my_id = str(int( log_data[0]['i'] / NUMBER_DEVICES)).zfill(LENGHT_DIRECTORY_DEVICES) other_id = str(int( interaction["o"] / NUMBER_DEVICES)).zfill(LENGHT_DIRECTORY_DEVICES) data_to_save1 = [ log_data[0]['i'], interaction["o"], interaction["w"], int(interaction["w"] + interaction["t"]), interaction["t"], interaction["r"], str(interaction["s"]).replace(",", ".") ] data_to_save2 = [ interaction["o"], log_data[0]['i'], interaction["w"], int(interaction["w"] + interaction["t"]), interaction["t"], interaction["r"], str(interaction["s"]).replace(",", ".") ] if 'x' in interaction: data_to_save1.append( str(interaction['x']).replace(",", ".")) data_to_save2.append( str(interaction['x']).replace(",", ".")) else: data_to_save1.append('') data_to_save2.append('') if 'y' in interaction: data_to_save1.append( str(interaction['y']).replace(",", ".")) data_to_save2.append( str(interaction['y']).replace(",", ".")) else: data_to_save1.append('') data_to_save2.append('') data_to_save1.append(log_data[0]['p']) data_to_save2.append(log_data[0]['p']) hostname = gethostname() path_to_file1 = base_path.copy() path_to_file2 = base_path.copy() for index in range(0, 9, 2): path_to_file1.append( str(my_id[index] + my_id[index + 1])) path_to_file2.append( str(other_id[index] + other_id[index + 1])) path_to_file1.append(hostname) path_to_file2.append(hostname) path_to_file1 = "/".join(map(str, path_to_file1)) path_to_file2 = "/".join(map(str, path_to_file2)) if path_to_file1 not in cache: makedirs(path_to_file1) cache.add(path_to_file1) if path_to_file2 not in cache: makedirs(path_to_file2) cache.add(path_to_file2) path_to_file1 += batch_number path_to_file2 += batch_number csv1 = ",".join(map(str, data_to_save1)) + "\n" csv2 = ",".join(map(str, data_to_save2)) + "\n" if path_to_file1 in bulk_write: bulk_write[path_to_file1] = bulk_write[ path_to_file1] + csv1 else: bulk_write[path_to_file1] = csv1 if path_to_file2 in bulk_write: bulk_write[path_to_file2] = bulk_write[ path_to_file2] + csv2 else: bulk_write[path_to_file2] = csv2 else: for event in log_data[0]: if all(key in event for key in expected_keys): event_date = datetime.fromtimestamp(event.get("w")) base_path = [ base_dir, event_date.year, str(event_date.month).zfill(2), str(event_date.day).zfill(2), ] id_my = event.get("i") id_other = event.get("o") my_id = str(int(id_my / NUMBER_DEVICES)).zfill( LENGHT_DIRECTORY_DEVICES) other_id = str(int(id_other / NUMBER_DEVICES)).zfill( LENGHT_DIRECTORY_DEVICES) timestamp_start = event["w"] timestamp_end = event["w"] + event["t"] duration = event["t"] distance_type = D_MAP[str(event["d"])] rssi = event["r"] latitude = event["x"] longitude = event["y"] platform = event["p"] hostname = gethostname() path_to_file1 = base_path.copy() path_to_file2 = base_path.copy() for index in range(0, 9, 2): path_to_file1.append( str(my_id[index] + my_id[index + 1])) path_to_file2.append( str(other_id[index] + other_id[index + 1])) path_to_file1.append(hostname) path_to_file2.append(hostname) path_to_file1 = "/".join(map(str, path_to_file1)) path_to_file2 = "/".join(map(str, path_to_file2)) if path_to_file1 not in cache: makedirs(path_to_file1) cache.add(path_to_file1) if path_to_file2 not in cache: makedirs(path_to_file2) cache.add(path_to_file2) path_to_file1 += batch_number path_to_file2 += batch_number # my_id, other_id, timestamp_start, timestamp_end, interaction_lenght_in_seconds, # RSSI, distance_type, latitude, longitude, data_to_save1 = [ id_my, id_other, timestamp_start, timestamp_end, duration, rssi, distance_type, latitude, longitude, platform ] data_to_save2 = [ id_other, id_my, timestamp_start, timestamp_end, duration, rssi, distance_type, latitude, longitude, platform ] csv1 = ",".join(map(str, data_to_save1)) + "\n" csv2 = ",".join(map(str, data_to_save2)) + "\n" if path_to_file1 in bulk_write: bulk_write[path_to_file1] = bulk_write[ path_to_file1] + csv1 else: bulk_write[path_to_file1] = csv1 if path_to_file2 in bulk_write: bulk_write[path_to_file2] = bulk_write[ path_to_file2] + csv2 else: bulk_write[path_to_file2] = csv2 for path, csv in bulk_write.items(): if path in file_cache: f = file_cache[path] else: f = open(path, "a+") file_cache[path] = f f.write(csv) if len(file_cache) > 5000: f = file_cache.pop(sample(file_cache.keys(), 1)[0]) f.close()
def hosts(self): if "_hosts" not in self.__dict__ or self._hosts is None: self._hosts = [] if len(self._hosts) == 0 and isfile(self.hostlist): logger.debug("Getting hosts", len(self._hosts)) pos = 0 decoder = JSONDecoder() with open(self.hostlist) as file: document = file.read() while True: match = NOT_WHITESPACE.search(document, pos) if not match: break pos = match.start() obj, pos = decoder.raw_decode(document, pos) if "hosts" in obj.keys(): for host in [ host for host in obj["hosts"] if self.host_filter.search(host[1]) ]: logger.debug("Adding {}@{}:{}", host[3], host[1], host[2]) try: result = DelugeHost( host=host[1], port=host[2], username=host[3], password=host[4], ) self._hosts.append(result) except timeout: logger.error( "Timeout connecting to {}@{}:{}", host[3], host[1], host[2], ) except BrokenPipeError: logger.error( "Broken Pipe connecting to {}@{}:{}", host[3], host[1], host[2], ) except ConnectionAbortedError: logger.error( "Connection Aborted connecting to {}@{}:{}", host[3], host[1], host[2], ) except ConnectionRefusedError: logger.error( "Connection Refused connecting to {}@{}:{}", host[3], host[1], host[2], ) except ConnectionResetError: logger.error( "Connection Reset connecting to {}@{}:{}", host[3], host[1], host[2], ) except DelugeNotConnectedException: logger.error( "Connection to {}@{}:{} failed", host[3], host[1], host[2], ) logger.debug("Got {} hosts", len(self._hosts)) return self._hosts
def read(cls, input_string, schema, index=0): """Deserialise an input JSONString to a python object""" decoder = JSONDecoder() decoded, index = decoder.raw_decode(input_string, index) def decode(input, schema): # special case for handling result codes if schema.role == Schema.Role.EResult: if not isinstance(input, int): raise ValueError("Could not interpret " + str(input) + " as a result code") return Result(input) # special case for handling functions. note that only outputs (and return values) are read if schema.role == Schema.Role.EFunction: output_schemas = [ s[1] for s in schema.sub_schemas if (s[1].role is Schema.Role.EReturn) or ( s[1].role is Schema.Role.EOutput) ] return tuple( decode(element[0], element[1]) for element in zip(input, output_schemas)) # special case for handling callbacks. if schema.role == Schema.Role.ECallback: if len(input) != len(schema.sub_schemas): raise ValueError( "Could not interpret " + str(input) + " as callback parameter tuple of length " + str(len(schema.sub_schemas))) return tuple( decode(element[0], element[1][1]) for element in zip(input, schema.sub_schemas)) if schema.type == Schema.Type.EVoid: return None if schema.type == Schema.Type.EBool: if isinstance(input, bool): return input if isinstance(input, int): return False if input == 0 else True raise ValueError("Could not interpret " + str(input) + " as " + schema.type.name) if schema.type.value >= Schema.Type.EInt8.value and schema.type.value <= Schema.Type.EFloat64.value: if not isinstance(input, numbers.Number): raise ValueError("Could not interpret " + str(input) + " as " + schema.type.name) return input if schema.type == Schema.Type.EString: if not isinstance(input, string_types): raise ValueError("Could not interpret " + str(input) + " as " + schema.type.name) return input if schema.type == Schema.Type.EArray: if len(input) != schema.count: raise ValueError("Could not interpret " + str(input) + " as array of length " + schema.count) return tuple( decode(element, schema.sub_schemas[0][1]) for element in input) if schema.type == Schema.Type.EList: return [ decode(element, schema.sub_schemas[0][1]) for element in input ] if schema.type == Schema.Type.ETuple: if len(input) != len(schema.sub_schemas): raise ValueError("Could not interpret " + str(input) + " as tuple of length " + str(len(schema.sub_schemas))) return tuple( decode(element[0], element[1][1]) for element in zip(input, schema.sub_schemas)) if schema.type == Schema.Type.ENamedTuple: record = cls.schema_records.get(schema.type_name, None) if not record: raise ValueError("Could not interpret unknown type " + schema.type_name) output = record.object() if isinstance(input, dict): for names, sub_schema in zip(record.pythonic_names.items(), schema.sub_schemas): input_element = input.get(names[0], None) if input_element: setattr(output, names[1], decode(input_element, sub_schema[1])) else: if len(input) != len(record.pythonic_names): raise ValueError("Could not interpret " + str(input) + " as class with " + str(len(record.pythonic_names)) + " members") for input_element, names, sub_schema in zip( input, record.pythonic_names.items(), schema.sub_schemas): setattr(output, names[1], decode(input_element, sub_schema[1])) return output if schema.type.value >= Schema.Type.EEnum8.value and schema.type.value <= Schema.Type.EEnum32.value: record = cls.schema_records.get(schema.type_name, None) if not record: raise ValueError("Could not read unregistered enum type " + str(input)) return record.object["E" + input] if schema.type == Schema.Type.ERef: return decode(input, cls.schema_records[schema.type_name].schema) return decode(decoded, schema)
def genjson(fp=None, filename=None, bufsize=262144, maxbufsize=None): """Generate JSON entries from a file-like object. Parameters ---------- fp : file, optional A file-like object with a sequence of JSON entries. filename : str, optional Path to a file with JSON entries. The `filename` and `fp` arguments are mutually exclusive; only one of them must be provided. bufsize : int, optional Size of the buffer to be read from the file for decoding. When decoding fails, the buffer is enlarged up to `maxbufsize` or until the whole file is consumed. maxbufsize : int, optional Maximum allowed size of the internal read buffer. When specified, each JSON entry must be shorter than that. Returns ------- generator The generator of JSON entries. Raises ------ JSONDecodeError When JSON decoding fails or if some entry exceeds `maxbufsize`. """ from json import JSONDecoder if fp is None and filename is None: raise TypeError("must have either 'fp' or 'filename' arguments") if fp is not None and filename is not None: raise TypeError("cannot use both 'fp' and 'filename' arguments") if filename is not None: kw = {'bufsize': bufsize, 'maxbufsize': maxbufsize} with open(filename) as fp1: for jobj in genjson(fp=fp1, **kw): yield jobj return # here we have file-like object in fp maxbuflen = sys.maxsize if maxbufsize is None else maxbufsize decoder = JSONDecoder() buf = '' while True: chunk = fp.read(bufsize) buf += chunk # end of file after processing last entry if buf == '': break try: jobj, i = decoder.raw_decode(buf) except ValueError: if chunk == '': raise if len(buf) > maxbuflen: raise continue buf = buf[i:].lstrip() yield jobj return
import urllib2 import lxml.html as lh from bs4 import BeautifulSoup as BS import requests import json import simplejson import urllib from json import JSONDecoder line ='{"noCompatibilityBySpec":false,"status":{"id":1,"name":"SUCCESS"},"hsnTsnExpansion":false,"propertyNames":["FitmentComments","Year","Make","Model","Trim","Engine"],"propertyTypes":[2,2,2,2,2,2],"displayNames":["Notes","Year","Make","Model","Trim","Engine"],"data":[{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, Drive Type: 4WD"],"Model":["K5 Blazer"],"Year":["1981"],"Make":["Chevrolet"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, Drive Type: 4WD"],"Model":["Jimmy"],"Year":["1981"],"Make":["GMC"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K1500"],"Year":["1981"],"Make":["GMC"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K1500 Suburban"],"Year":["1981"],"Make":["GMC"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["Cherokee"],"Year":["1981"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["J10"],"Year":["1981"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, Except Splined Drive Hub"],"Model":["Wagoneer"],"Year":["1981"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K10"],"Year":["1980"],"Make":["Chevrolet"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K10 Suburban"],"Year":["1980"],"Make":["Chevrolet"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, Drive Type: 4WD"],"Model":["K5 Blazer"],"Year":["1980"],"Make":["Chevrolet"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, Drive Type: 4WD"],"Model":["Jimmy"],"Year":["1980"],"Make":["GMC"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K1500"],"Year":["1980"],"Make":["GMC"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K1500 Suburban"],"Year":["1980"],"Make":["GMC"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, 7/8" Drive Flange"],"Model":["Cherokee"],"Year":["1980"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, 6 Stud Rotors, Steering Drive Flange"],"Model":["CJ5"],"Year":["1980"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, Steering Drive Flange, 6 Stud Rotors"],"Model":["CJ7"],"Year":["1980"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, 7/8" Drive Flange"],"Model":["J10"],"Year":["1980"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, 7/8" Drive Flange"],"Model":["Wagoneer"],"Year":["1980"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K10"],"Year":["1979"],"Make":["Chevrolet"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K10 Suburban"],"Year":["1979"],"Make":["Chevrolet"]}],"errors":[],"uniqueVehicle":false,"fitsDiag":[],"pageInfo":{"currentPageNo":5,"totalRecordCount":176,"totalPageCount":9,"pageRecordCount":20}}' decoder = JSONDecoder() s_len = len(line) objs = [] end = 0 while end != s_len: obj, end = decoder.raw_decode(line, idx=end) objs.append(obj) print objs print "JSON parsing example: ", objs[0]['data'][0]['Model']
from json import JSONDecoder j = JSONDecoder() a = '128931233' b = "472389423" if id(a) < id(b): x = a y = b else: x = b y = a diff = id(x) - id(y) try: j.raw_decode(y, diff) print("Vulnerable") except: print("Not vulnerable")