예제 #1
0
    def _write_control_event(self, text):
        text = textwrap_indent(text, '* ', predicate=lambda line: True)
        self._control_record.block_file.write(
            text.encode('utf-8', errors='surrogateescape'))

        if not text.endswith('\n'):
            self._control_record.block_file.write(b'\n')
예제 #2
0
파일: warc.py 프로젝트: asergi/wpull
    def _write_control_event(self, text):
        text = textwrap_indent(text, '* ', predicate=lambda line: True)
        self._control_record.block_file.write(
            text.encode('utf-8', errors='surrogateescape')
        )

        if not text.endswith('\n'):
            self._control_record.block_file.write(b'\n')
예제 #3
0
    def response_control_data(self, data):
        text = textwrap_indent(data.decode('utf-8', errors='surrogateescape'),
                               '< ',
                               predicate=lambda line: True)
        self._control_record.block_file.write(
            text.encode('utf-8', errors='surrogateescape'))

        if not data.endswith(b'\n'):
            self._control_record.block_file.write(b'\n')
예제 #4
0
파일: warc.py 프로젝트: asergi/wpull
    def response_control_data(self, data):
        text = textwrap_indent(
            data.decode('utf-8', errors='surrogateescape'),
            '< ', predicate=lambda line: True
        )
        self._control_record.block_file.write(
            text.encode('utf-8', errors='surrogateescape')
        )

        if not data.endswith(b'\n'):
            self._control_record.block_file.write(b'\n')
예제 #5
0
파일: misc.py 프로젝트: Raniac/NEURO-LEARN
def dict_diff(dold, dnew, indent=0):
    """Helper to log what actually changed from old to new values of
    dictionaries.

    typical use -- log difference for hashed_inputs
    """
    # First check inputs, since they usually are lists of tuples
    # and dicts are required.
    if isinstance(dnew, list):
        dnew = dict(dnew)
    if isinstance(dold, list):
        dold = dict(dold)

    # Compare against hashed_inputs
    # Keys: should rarely differ
    new_keys = set(dnew.keys())
    old_keys = set(dold.keys())

    diff = []
    if new_keys - old_keys:
        diff += ["  * keys not previously seen: %s" % (new_keys - old_keys)]

    if old_keys - new_keys:
        diff += ["  * keys not presently seen: %s" % (old_keys - new_keys)]

    # Add topical message
    if diff:
        diff.insert(0, "Dictionaries had differing keys:")

    diffkeys = len(diff)

    # Values in common keys would differ quite often,
    # so we need to join the messages together
    for k in new_keys.intersection(old_keys):
        try:
            new, old = dnew[k], dold[k]
            same = new == old
            if not same:
                # Since JSON does not discriminate between lists and
                # tuples, we might need to cast them into the same type
                # as the last resort.  And lets try to be more generic
                same = old.__class__(new) == old
        except Exception:
            same = False
        if not same:
            diff += ["  * %s: %r != %r" % (k, dnew[k], dold[k])]

    if len(diff) > diffkeys:
        diff.insert(diffkeys, "Some dictionary entries had differing values:")

    return textwrap_indent('\n'.join(diff), ' ' * indent)