def iterload(string_or_fp, cls=json.JSONDecoder, **kwargs):
"""
Source: https://stackoverflow.com/questions/6886283/how-i-can-i-lazily-read-multiple-json-values-from-a-file-stream-in-python
"""
if isinstance(string_or_fp, str):
string = str(string_or_fp)
else:
string = string_or_fp.read()
decoder = cls(**kwargs)
idx = WHITESPACE.match(string, 0).end()
while idx < len(string):
retry_count = 3
obj, end = None, None
while retry_count > 0:
try:
obj, end = decoder.raw_decode(string, idx)
break
except Exception as e:
retry_count -= 1
yield obj
if obj is None:
idx = WHITESPACE.match(string, idx + 1).end()
else:
idx = WHITESPACE.match(string, end).end()
def collect_loop(self, conn, events):
global thr_summary, thr_slowest
string = ""
while True:
self.mi_refresh_sub()
try:
new = conn.recv(1024).decode('utf-8')
except socket.timeout:
new = ""
if threading.current_thread().stopped():
self.mi_unsub()
break
if not new:
continue
string += new
decoder = json.JSONDecoder()
idx = WHITESPACE.match(string, 0).end()
while idx < len(string):
try:
obj, end = decoder.raw_decode(string, idx)
except json.decoder.JSONDecodeError:
# partial JSON -- just let it accumulate
break
if 'params' not in obj:
string = string[end:]
continue
params = obj['params']
# only process threshold events we're interested in
if events is None or \
any(params['source'].startswith(e) for e in events):
if 'extra' not in params:
params['extra'] = "<unknown>"
if not self.skip_summ:
try:
thr_summary[(params['extra'],
params['source'])] += 1
except:
thr_summary[(params['extra'],
params['source'])] = 1
bisect.insort(
thr_slowest,
(-params['time'], params['extra'], params['source']))
thr_slowest = thr_slowest[:3]
string = string[end:]
idx = WHITESPACE.match(string, 0).end()
def evil_decode(docbytes):
decoder = json.JSONDecoder()
idx = WHITESPACE.match(docbytes, 0).end()
while idx < len(docbytes):
try:
obj, end = decoder.raw_decode(docbytes, idx)
yield obj
idx = WHITESPACE.match(docbytes, end).end()
except ValueError as e:
print('docbytes:\n', docbytes)
raise
def json_iterload(string_or_fp, cls=json.JSONDecoder, **kwargs):
if isinstance(string_or_fp, io.IOBase):
string = string_or_fp.read()
else:
string = str(string_or_fp)
decoder = cls(**kwargs)
idx = WHITESPACE.match(string, 0).end()
while idx < len(string):
obj, end = decoder.raw_decode(string, idx)
yield obj
idx = WHITESPACE.match(string, end).end()
def iterload(fp, cls=json.JSONDecoder, **kwargs):
if (isinstance(fp, io.TextIOBase) or isinstance(fp, io.BufferedIOBase)
or isinstance(fp, io.RawIOBase) or isinstance(fp, io.IOBase)):
string = fp.read()
else:
string = str(fp)
decoder = cls(**kwargs)
idx = WHITESPACE.match(string, 0).end()
while idx < len(string):
obj, end = decoder.raw_decode(string, idx)
yield obj
idx = WHITESPACE.match(string, end).end()
def iterload(self, string_or_fp, cls=json.JSONDecoder, **kwargs):
if isinstance(string_or_fp, file):
string = string_or_fp.read()
else:
string = str(string_or_fp)
decoder = cls(**kwargs)
idx = WHITESPACE.match(string, 0).end()
while idx < len(string):
obj, end = decoder.raw_decode(string, idx)
# yield obj
return obj
idx = WHITESPACE.match(string, end).end()
def iterload(string_or_fp, cls=json.JSONDecoder, **kwargs):
if isinstance(string_or_fp, file):
string = string_or_fp.read()
else:
string = str(string_or_fp)
# set args for decoder if specified
decoder = cls(**kwargs)
# skip over whitespace to beginning of first obj
idx = WHITESPACE.match(string, 0).end()
while idx < len(string):
# object and index of the end of obj in string
obj, end = decoder.raw_decode(string, idx)
yield obj
# skip whitespace till next object
idx = WHITESPACE.match(string, end).end()
def parse_iter(s):
size = len(s)
decoder = json.JSONDecoder()
end = 0
while True:
idx = WHITESPACE.match(data).end()
if end + idx >= size:
break
ob, end = decoder.raw_decode(data, end + idx)
yield ob
def loads_iter(s):
size = len(s)
decoder = json.JSONDecoder()
end = 0
while True:
idx = WHITESPACE.match(s[end:]).end()
i = end + idx
if i >= size:
break
ob, end = decoder.raw_decode(s, i)
yield ob
def loads_iter(s):
size = len(s)
decoder = json.JSONDecoder()
end = 0
while True:
idx = WHITESPACE.match(s[end:]).end()
i = end + idx
if i >= size:
break
ob, end = decoder.raw_decode(s, i)
yield ob
def _load_buffered(stream):
s = stream.read()
size = len(s)
decoder = json.JSONDecoder(object_pairs_hook=OrderedDict)
end = 0
while True:
idx = WHITESPACE.match(s[end:]).end()
i = end + idx
if i >= size:
break
ob, end = decoder.raw_decode(s, i)
yield ob
def decode_multiple(self, data):
# force input data into string
string = str(data)
self.logger.debug("Decoding data from Core: %s" % string)
# create decoder to identify json strings
decoder = json.JSONDecoder()
idx = WHITESPACE.match(string, 0).end()
self.logger.debug("Decode WHITESPACE match: %d" % idx)
ls = len(string)
result = []
while idx < ls:
try:
obj, end = decoder.raw_decode(string, idx)
self.logger.debug("JSON object(%d, %d): %s" % (idx, end, obj))
result.append(obj)
except ValueError, e:
self.logger.debug("ValueError exception: %s" % e)
#to force functione exit
idx = ls
else:
idx = WHITESPACE.match(string, end).end()
def _load_unbuffered(stream):
buf = deque([], maxlen=100)
first_err = None
decoder = json.JSONDecoder(object_pairs_hook=OrderedDict)
for line in stream:
buf.append(line)
try:
body = "".join(buf)
if len(body.strip()) > 0:
idx = WHITESPACE.match(body).end()
ob, end = decoder.raw_decode(body, idx)
first_err = None
yield ob
buf.clear()
buf.append(body[end:])
except JSONDecodeError as e:
if first_err is None:
first_err = e
if first_err is not None:
raise first_err
try:
shutil.rmtree("translators")
except:
pass
os.mkdir("translators")
indexfile = open("translators/getTranslators", "w")
firstFile = True
decoder = json.JSONDecoder()
for filename in glob.iglob('../dependencies/zotero-connectors/src/zotero/translators/*.js'):
file = open(filename, "r")
filecontents = file.read()
file.close()
start = WHITESPACE.match(filecontents, 0).end()
translatordata, endindex = decoder.raw_decode(filecontents, start)
shutil.copyfile(filename, "translators/" + translatordata["translatorID"])
if firstFile:
indexfile.write("[")
firstFile = False
else:
indexfile.write(",")
json.dump(translatordata, indexfile)
indexfile.write("]")
indexfile.close()