Esempi in Python per safe_size

Esempio n. 1

File: s3.py Progetto: klahnakoski/MoTreeherder

    def read(self, key):
        source = self.get_meta(key)

        try:
            json = safe_size(source)
        except Exception, e:
            Log.error(READ_ERROR, e)

Esempio n. 2

    def read(self, key):
        source = self.get_meta(key)

        try:
            json = safe_size(source)
        except Exception, e:
            Log.error(READ_ERROR, e)

Esempio n. 3

File: convert.py Progetto: davehunt/ActiveData

def zip2bytes(compressed):
    """
    UNZIP DATA
    """
    if hasattr(compressed, "read"):
        return gzip.GzipFile(fileobj=compressed, mode='r')

    buff = BytesIO(compressed)
    archive = gzip.GzipFile(fileobj=buff, mode='r')
    return safe_size(archive)

Esempio n. 4

def zip2bytes(compressed):
    """
    UNZIP DATA
    """
    if hasattr(compressed, "read"):
        return gzip.GzipFile(fileobj=compressed, mode='r')

    buff = BytesIO(compressed)
    archive = gzip.GzipFile(fileobj=buff, mode='r')
    from pyLibrary.env.big_data import safe_size
    return safe_size(archive)

Esempio n. 5

File: convert.py Progetto: klahnakoski/SpotManager

def zip2bytes(compressed):
    """
    UNZIP DATA
    """
    if hasattr(compressed, "read"):
        return gzip.GzipFile(fileobj=compressed, mode='r')

    buff = BytesIO(compressed)
    archive = gzip.GzipFile(fileobj=buff, mode='r')
    from pyLibrary.env.big_data import safe_size
    return safe_size(archive)

Esempio n. 6

    def read(self, key):
        source = self.get_meta(key)

        try:
            json = safe_size(source)
        except Exception as e:
            Log.error(READ_ERROR, e)

        if json == None:
            return None

        if source.key.endswith(".zip"):
            json = _unzip(json)
        elif source.key.endswith(".gz"):
            json = convert.zip2bytes(json)

        return json.decode('utf8')

Esempio n. 7

File: s3.py Progetto: rv404674/TUID

    def read(self, key):
        source = self.get_meta(key)

        try:
            json = safe_size(source)
        except Exception as e:
            Log.error(READ_ERROR, e)

        if json == None:
            return None

        if source.key.endswith(".zip"):
            json = _unzip(json)
        elif source.key.endswith(".gz"):
            json = convert.zip2bytes(json)

        return utf82unicode(json)

Esempio n. 8

File: http.py Progetto: klahnakoski/intermittents

    def all_content(self):
        # Response.content WILL LEAK MEMORY (?BECAUSE OF PYPY"S POOR HANDLING OF GENERATORS?)
        # THE TIGHT, SIMPLE, LOOP TO FILL blocks PREVENTS THAT LEAK
        if self._cached_content is None:
            def read(size):
                if self.raw._fp.fp is not None:
                    return self.raw.read(amt=size, decode_content=True)
                else:
                    self.close()
                    return None

            self._cached_content = safe_size(Dict(read=read))

        if hasattr(self._cached_content, "read"):
            self._cached_content.seek(0)

        return self._cached_content

Esempio n. 9

File: http.py Progetto: vpathak2019/jx-sqlite

    def all_content(self):
        # response.content WILL LEAK MEMORY (?BECAUSE OF PYPY"S POOR HANDLING OF GENERATORS?)
        # THE TIGHT, SIMPLE, LOOP TO FILL blocks PREVENTS THAT LEAK
        if self._content is not False:
            self._cached_content = self._content
        elif self._cached_content is None:
            def read(size):
                if self.raw._fp.fp is not None:
                    return self.raw.read(amt=size, decode_content=True)
                else:
                    self.close()
                    return None

            self._cached_content = safe_size(Data(read=read))

        if hasattr(self._cached_content, "read"):
            self._cached_content.seek(0)

        return self._cached_content

Esempio n. 10

File: s3.py Progetto: klahnakoski/Activedata-ETL

    def read_lines(self, key):
        source = self.get_meta(key)
        if source is None:
            Log.error("{{key}} does not exist",  key= key)
        if source.size < MAX_STRING_SIZE:
            if source.key.endswith(".gz"):
                return GzipLines(source.read())
            else:
                return convert.utf82unicode(source.read()).split("\n")

        if source.key.endswith(".gz"):
            bytes = safe_size(source)
            if isinstance(bytes, str):
                buff = BytesIO(bytes)
            else:
                # SWAP OUT FILE REFERENCE
                bytes.file, buff = None, bytes.file
            archive = gzip.GzipFile(fileobj=buff, mode='r')
            return LazyLines(archive)
        else:
            return LazyLines(source)

Esempio n. 11

    def read_lines(self, key):
        source = self.get_meta(key)
        if source is None:
            Log.error("{{key}} does not exist", key=key)
        if source.size < MAX_STRING_SIZE:
            if source.key.endswith(".gz"):
                return GzipLines(source.read())
            else:
                return convert.utf82unicode(source.read()).split("\n")

        if source.key.endswith(".gz"):
            bytes = safe_size(source)
            if isinstance(bytes, str):
                buff = BytesIO(bytes)
            else:
                # SWAP OUT FILE REFERENCE
                bytes.file, buff = None, bytes.file
            archive = gzip.GzipFile(fileobj=buff, mode='r')
            return LazyLines(archive)
        else:
            return LazyLines(source)

Esempio n. 12

 def read_bytes(self, key):
     source = self.get_meta(key)
     return safe_size(source)

Esempio n. 13

File: s3.py Progetto: klahnakoski/MoTreeherder

 def read_bytes(self, key):
     source = self.get_meta(key)
     return safe_size(source)