Beispiel #1
0
def iterload(string_or_fp, cls=json.JSONDecoder, **kwargs):
    """
    Source: https://stackoverflow.com/questions/6886283/how-i-can-i-lazily-read-multiple-json-values-from-a-file-stream-in-python
    """
    if isinstance(string_or_fp, str):
        string = str(string_or_fp)
    else:
        string = string_or_fp.read()

    decoder = cls(**kwargs)
    idx = WHITESPACE.match(string, 0).end()
    while idx < len(string):
        retry_count = 3
        obj, end = None, None
        while retry_count > 0:
            try:
                obj, end = decoder.raw_decode(string, idx)
                break
            except Exception as e:
                retry_count -= 1

        yield obj
        if obj is None:
            idx = WHITESPACE.match(string, idx + 1).end()
        else:
            idx = WHITESPACE.match(string, end).end()
Beispiel #2
0
    def collect_loop(self, conn, events):
        global thr_summary, thr_slowest

        string = ""
        while True:
            self.mi_refresh_sub()

            try:
                new = conn.recv(1024).decode('utf-8')
            except socket.timeout:
                new = ""

            if threading.current_thread().stopped():
                self.mi_unsub()
                break

            if not new:
                continue

            string += new

            decoder = json.JSONDecoder()
            idx = WHITESPACE.match(string, 0).end()
            while idx < len(string):
                try:
                    obj, end = decoder.raw_decode(string, idx)
                except json.decoder.JSONDecodeError:
                    # partial JSON -- just let it accumulate
                    break

                if 'params' not in obj:
                    string = string[end:]
                    continue

                params = obj['params']

                # only process threshold events we're interested in
                if events is None or \
                        any(params['source'].startswith(e) for e in events):
                    if 'extra' not in params:
                        params['extra'] = "<unknown>"

                    if not self.skip_summ:
                        try:
                            thr_summary[(params['extra'],
                                         params['source'])] += 1
                        except:
                            thr_summary[(params['extra'],
                                         params['source'])] = 1

                    bisect.insort(
                        thr_slowest,
                        (-params['time'], params['extra'], params['source']))
                    thr_slowest = thr_slowest[:3]

                string = string[end:]
                idx = WHITESPACE.match(string, 0).end()
Beispiel #3
0
def evil_decode(docbytes):
    decoder = json.JSONDecoder()
    idx = WHITESPACE.match(docbytes, 0).end()
    while idx < len(docbytes):
        try:
            obj, end = decoder.raw_decode(docbytes, idx)
            yield obj
            idx = WHITESPACE.match(docbytes, end).end()
        except ValueError as e:
            print('docbytes:\n', docbytes)
            raise
Beispiel #4
0
def json_iterload(string_or_fp, cls=json.JSONDecoder, **kwargs):
    if isinstance(string_or_fp, io.IOBase):
        string = string_or_fp.read()
    else:
        string = str(string_or_fp)

    decoder = cls(**kwargs)
    idx = WHITESPACE.match(string, 0).end()
    while idx < len(string):
        obj, end = decoder.raw_decode(string, idx)
        yield obj
        idx = WHITESPACE.match(string, end).end()
Beispiel #5
0
def iterload(fp, cls=json.JSONDecoder, **kwargs):
    if (isinstance(fp, io.TextIOBase) or isinstance(fp, io.BufferedIOBase)
            or isinstance(fp, io.RawIOBase) or isinstance(fp, io.IOBase)):
        string = fp.read()
    else:
        string = str(fp)

    decoder = cls(**kwargs)
    idx = WHITESPACE.match(string, 0).end()
    while idx < len(string):
        obj, end = decoder.raw_decode(string, idx)
        yield obj
        idx = WHITESPACE.match(string, end).end()
Beispiel #6
0
    def iterload(self, string_or_fp, cls=json.JSONDecoder, **kwargs):
        if isinstance(string_or_fp, file):
            string = string_or_fp.read()
        else:
            string = str(string_or_fp)

        decoder = cls(**kwargs)
        idx = WHITESPACE.match(string, 0).end()
        while idx < len(string):
            obj, end = decoder.raw_decode(string, idx)
            # yield obj
            return obj
            idx = WHITESPACE.match(string, end).end()
Beispiel #7
0
def iterload(string_or_fp, cls=json.JSONDecoder, **kwargs):
    if isinstance(string_or_fp, file):
        string = string_or_fp.read()
    else:
        string = str(string_or_fp)
    # set args for decoder if specified
    decoder = cls(**kwargs)
    # skip over whitespace to beginning of first obj
    idx = WHITESPACE.match(string, 0).end()
    while idx < len(string):
        # object and index of the end of obj in string
        obj, end = decoder.raw_decode(string, idx)
        yield obj
        # skip whitespace till next object
        idx = WHITESPACE.match(string, end).end()
Beispiel #8
0
def parse_iter(s):
    size = len(s)
    decoder = json.JSONDecoder()

    end = 0
    while True:
        idx = WHITESPACE.match(data).end()
        if end + idx >= size:
            break
        ob, end = decoder.raw_decode(data, end + idx)
        yield ob
Beispiel #9
0
def loads_iter(s):
    size = len(s)
    decoder = json.JSONDecoder()

    end = 0
    while True:
        idx = WHITESPACE.match(s[end:]).end()
        i = end + idx
        if i >= size:
            break
        ob, end = decoder.raw_decode(s, i)
        yield ob
Beispiel #10
0
def loads_iter(s):
    size = len(s)
    decoder = json.JSONDecoder()

    end = 0
    while True:
        idx = WHITESPACE.match(s[end:]).end()
        i = end + idx
        if i >= size:
            break
        ob, end = decoder.raw_decode(s, i)
        yield ob
Beispiel #11
0
def _load_buffered(stream):
    s = stream.read()
    size = len(s)
    decoder = json.JSONDecoder(object_pairs_hook=OrderedDict)

    end = 0
    while True:
        idx = WHITESPACE.match(s[end:]).end()
        i = end + idx
        if i >= size:
            break
        ob, end = decoder.raw_decode(s, i)
        yield ob
Beispiel #12
0
	def decode_multiple(self, data):
		# force input data into string
		string = str(data)
		self.logger.debug("Decoding data from Core: %s" % string)

		# create decoder to identify json strings
		decoder = json.JSONDecoder()
		idx = WHITESPACE.match(string, 0).end()
		self.logger.debug("Decode WHITESPACE match: %d" % idx)

		ls = len(string)
		result = []
		while idx < ls:
			try:
				obj, end = decoder.raw_decode(string, idx)
				self.logger.debug("JSON object(%d, %d): %s" % (idx, end, obj))
				result.append(obj)
			except ValueError, e:
				self.logger.debug("ValueError exception: %s" % e)

				#to force functione exit
				idx = ls
			else:
				idx = WHITESPACE.match(string, end).end()
Beispiel #13
0
def _load_unbuffered(stream):
    buf = deque([], maxlen=100)
    first_err = None

    decoder = json.JSONDecoder(object_pairs_hook=OrderedDict)

    for line in stream:
        buf.append(line)
        try:
            body = "".join(buf)
            if len(body.strip()) > 0:
                idx = WHITESPACE.match(body).end()
                ob, end = decoder.raw_decode(body, idx)
                first_err = None
                yield ob
                buf.clear()
                buf.append(body[end:])
        except JSONDecodeError as e:
            if first_err is None:
                first_err = e

    if first_err is not None:
        raise first_err
try:
    shutil.rmtree("translators")
except:
    pass

os.mkdir("translators")

indexfile = open("translators/getTranslators", "w")
firstFile = True

decoder = json.JSONDecoder()

for filename in glob.iglob('../dependencies/zotero-connectors/src/zotero/translators/*.js'):
    file = open(filename, "r")
    filecontents = file.read()
    file.close()

    start = WHITESPACE.match(filecontents, 0).end()
    translatordata, endindex = decoder.raw_decode(filecontents, start)
    shutil.copyfile(filename, "translators/" + translatordata["translatorID"])

    if firstFile:
        indexfile.write("[")
        firstFile = False
    else:
        indexfile.write(",")
    json.dump(translatordata, indexfile)

indexfile.write("]")
indexfile.close()