def run(self):
logging.info("Resolving oplog for host %s:%s to max timestamp: %s" % (self.host, self.port, self.max_end_ts))
try:
if self.dump_gzip:
tailed_oplog_fh = GzipFile(self.tailed_oplog_file)
mongodump_oplog_fh = GzipFile(self.mongodump_oplog_file, 'a+')
else:
tailed_oplog_fh = open(self.tailed_oplog_file)
mongodump_oplog_fh = open(self.mongodump_oplog_file, 'a+')
for change in decode_file_iter(tailed_oplog_fh):
if 'ts' in change:
ts = change['ts']
if ts > self.mongodump_oplog_last_ts or self.mongodump_oplog_last_ts is None:
if ts < self.max_end_ts:
mongodump_oplog_fh.write(BSON.encode(change))
self.changes += 1
self.last_ts = ts
elif ts > self.max_end_ts:
break
tailed_oplog_fh.close()
mongodump_oplog_fh.flush()
mongodump_oplog_fh.close()
except Exception, e:
logging.fatal("Resolving of oplogs failed! Error: %s" % e)
raise e
def gzip_generator(string_generator):
"""Return generator for gzipping given string generator.
Example:
>>> import StringIO
>>> z = ''.join(gzip_generator(iter(['hello,', ' ', 'world!'])))
>>> ''.join(gunzip_generator(StringIO.StringIO(z)))
'hello, world!'
"""
# Use gzip and not zlib to make proper gzip header.
buffer = StringIO()
gzip = GzipFile(fileobj=buffer, mode='w')
# Yield header
yield buffer.getvalue()
buffer.truncate(0)
for string in string_generator:
gzip.write(string)
gzip.flush()
yield buffer.getvalue()
buffer.truncate(0)
# Flush
gzip.close()
yield buffer.getvalue()
def compress_string(s):
# avg_block_size is acutally the reciporical of the average
# intended interflush distance.
rnd = Random(s)
flushes_remaining = FLUSH_LIMIT
if len(s) < AVERAGE_SPAN_BETWEEN_FLUSHES * APPROX_MIN_FLUSHES:
avg_block_size = APPROX_MIN_FLUSHES / float(len(s) + 1)
else:
avg_block_size = 1.0 / AVERAGE_SPAN_BETWEEN_FLUSHES
s = StringIO(s)
zbuf = BytesIO()
zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
chunk = s.read(MIN_INTERFLUSH_INTERVAL +
int(rnd.expovariate(avg_block_size)))
while chunk and flushes_remaining:
zfile.write(chunk)
zfile.flush()
flushes_remaining -= 1
chunk = s.read(MIN_INTERFLUSH_INTERVAL +
int(rnd.expovariate(avg_block_size)))
zfile.write(chunk)
zfile.write(s.read())
zfile.close()
return zbuf.getvalue()
def to_gzip(data):
iobuffer = StringIO()
gzip_mem_object = GzipFile(mode='wb', compresslevel=6, fileobj=iobuffer)
gzip_mem_object.write(data)
gzip_mem_object.flush()
gzip_mem_object.close()
return iobuffer.getvalue()
def gzip_generator(string_generator):
"""Return generator for gzipping given string generator.
Example:
>>> import StringIO
>>> z = ''.join(gzip_generator(iter(['hello,', ' ', 'world!'])))
>>> ''.join(gunzip_generator(StringIO.StringIO(z)))
'hello, world!'
"""
# Use gzip and not zlib to make proper gzip header.
buffer = StringIO()
gzip = GzipFile(fileobj=buffer, mode='w')
# Yield header
yield buffer.getvalue()
buffer.truncate(0)
for string in string_generator:
gzip.write(string)
gzip.flush()
yield buffer.getvalue()
buffer.truncate(0)
# Flush
gzip.close()
yield buffer.getvalue()
class GzipWrap(object):
def __init__(self, input, filename=None, compresslevel=1):
self.input = iter(input)
self.buffer = ""
self.zipper = GzipFile(filename,
mode="wb",
fileobj=self,
compresslevel=compresslevel,
mtime=0)
self.is_done = False
def read(self, size=-1):
if size is None or size < 0:
raise Exception(
"Call to GzipWrap with unbound size will result in poor performance"
)
# If the buffer already has enough bytes, then simply pop them off of
# the beginning and return them.
if len(self.buffer) >= size or self.is_done:
ret = self.buffer[0:size]
self.buffer = self.buffer[size:]
return ret
# Otherwise, zip the input until we have enough bytes.
while True:
# Attempt to retrieve the next bytes to write.
is_done = False
input_size = 0
input_buffer = ""
while input_size < GZIP_BUFFER_SIZE:
try:
s = self.input.next()
input_buffer += s
input_size = input_size + len(s)
except StopIteration:
is_done = True
break
self.zipper.write(input_buffer)
if is_done:
self.zipper.flush()
self.zipper.close()
self.is_done = True
if len(self.buffer) >= size or is_done:
ret = self.buffer[0:size]
self.buffer = self.buffer[size:]
return ret
def flush(self):
pass
def write(self, data):
self.buffer += data
def close(self):
self.input.close()
def compress_string(s):
# avg_block_size is acutally the reciporical of the average
# intended interflush distance.
rnd = Random(s)
flushes_remaining = FLUSH_LIMIT
if len(s) < AVERAGE_SPAN_BETWEEN_FLUSHES * APPROX_MIN_FLUSHES:
avg_block_size = APPROX_MIN_FLUSHES / float(len(s) + 1)
else:
avg_block_size = 1.0 / AVERAGE_SPAN_BETWEEN_FLUSHES
s = StringIO(s) if isinstance(s, six.text_type) else BytesIO(s)
zbuf = BytesIO()
zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
chunk = s.read(MIN_INTERFLUSH_INTERVAL + int(rnd.expovariate(avg_block_size)))
while chunk and flushes_remaining:
zfile.write(chunk)
zfile.flush()
flushes_remaining -= 1
chunk = s.read(MIN_INTERFLUSH_INTERVAL + int(rnd.expovariate(avg_block_size)))
zfile.write(chunk)
zfile.write(s.read())
zfile.close()
return zbuf.getvalue()
def write_to(self, out, newline='\x0D\x0A', gzip=False):
if gzip:
out = GzipFile(fileobj=out)
self._write_to(out, newline)
if gzip:
out.flush()
out.close()
def to_gzip(data):
iobuffer = StringIO()
gzip_mem_object = GzipFile(mode='wb', compresslevel=6, fileobj=iobuffer)
gzip_mem_object.write(data)
gzip_mem_object.flush()
gzip_mem_object.close()
return iobuffer.getvalue()
def write_to(self, out, newline='\x0D\x0A', gzip=False):
if gzip:
out = GzipFile(fileobj=out)
self._write_to(out, newline)
if gzip:
out.flush()
out.close()
def GET(self):
web.header('content-encoding', 'gzip')
out = StringIO()
f = GzipFile(fileobj=out, mode='w', compresslevel=6)
name = web.input(name='').name
f.write(SITE % name)
f.flush()
return out.getvalue()
def GET(self):
web.header('content-encoding','gzip')
out = StringIO()
f = GzipFile(fileobj=out, mode='w', compresslevel=6)
name = web.input(name='').name
f.write(SITE % name)
f.flush()
return out.getvalue()
def gzip_file(self, fff, compresslevel=6):
with closing(StringIO()) as strio:
gz = GzipFile(mode='wb',
compresslevel=compresslevel,
fileobj=strio)
gz.write(fff)
gz.flush()
gz.close()
return strio.getvalue()
def GET(self):
web.header('content-encoding','gzip')
out = StringIO()
f = GzipFile(fileobj=out, mode='w', compresslevel=6)
name = web.input(name='').name
name = name.replace('+', ' ')
f.write((SITE % name).replace('f675d2395f243c89', 'correct_horse_battery_staple'))
f.flush()
return out.getvalue()
class NBTFile(TAG_Compound):
"""Represents an NBT file object"""
def __init__(self, filename=None, mode=None, buffer=None):
super(NBTFile,self).__init__()
self.__class__.__name__ = "TAG_Compound"
self.filename = filename
self.type = TAG_Byte(self.id)
#make a file object
if filename:
self.file = GzipFile(filename, mode)
elif buffer:
self.file = buffer
else:
self.file = None
#parse the file given intitially
if self.file:
self.parse_file()
if filename and 'close' in dir(self.file):
self.file.close()
self.file = None
def parse_file(self, filename=None, buffer=None):
if filename:
self.file = GzipFile(filename, 'rb')
if buffer:
self.file = buffer
if self.file:
type = TAG_Byte(buffer=self.file)
if type.value == self.id:
name = TAG_String(buffer=self.file).value
self._parse_buffer(self.file)
self.name = name
self.file.close()
else:
raise ValueError("First record is not a Compound Tag")
else: ValueError("need a file!")
def write_file(self, filename=None, buffer=None):
if buffer:
self.file = buffer
elif filename:
self.file = GzipFile(filename, "wb")
elif self.filename:
self.file = GzipFile(self.filename, "wb")
elif not self.file:
raise ValueError("Need to specify either a filename or a file")
#Render tree to file
TAG_Byte(self.id)._render_buffer(self.file)
TAG_String(self.name)._render_buffer(self.file)
self._render_buffer(self.file)
#make sure the file is complete
if 'flush' in dir(self.file):
self.file.flush()
if filename and 'close' in dir(self.file):
self.file.close()
def GET(self):
web.header('content-encoding', 'gzip')
out = StringIO()
f = GzipFile(fileobj=out, mode='w', compresslevel=6)
name = web.input(name='').name
name = name.replace('+', ' ')
f.write((SITE % name).replace('f675d2395f243c89',
'correct_horse_battery_staple'))
f.flush()
return out.getvalue()
def write_sample_data(self, file, compress=False):
if compress:
file = GzipFile(mode='w', fileobj=file)
for line in fragmented_message_type_8:
file.write(bytes(line, "ascii"))
file.write(newline)
file.write(bytes("garbage data", "ascii"))
file.write(newline)
file.write(bytes(message_type_1, "ascii"))
file.write(newline)
file.flush()
def write_sample_data(self, file, compress=False):
if compress:
file = GzipFile(mode='w', fileobj=file)
for line in fragmented_message_type_8:
file.write(bytes(line, "ascii"))
file.write(newline)
file.write(bytes("garbage data", "ascii"))
file.write(newline)
file.write(bytes(message_type_1, "ascii"))
file.write(newline)
file.flush()
def compress_sequence(sequence):
buf = StreamingBuffer()
zfile = GzipFile(mode='wb', compresslevel=6, fileobj=buf)
# Output headers...
yield buf.read()
for item in sequence:
zfile.write(item)
zfile.flush()
yield buf.read()
zfile.close()
yield buf.read()
class GzipEncoding(Encoding):
def __init__(self, compression_level=6):
super()
self._encoder_buffer = io.BytesIO()
self._encoder = None
self._decoder_buffer = io.BytesIO()
self._decoder = zlib.decompressobj(16 + zlib.MAX_WBITS)
self._compression_level = compression_level
def name(self):
return 'gzip'
def encode(self, data):
data = data.encode('utf-8')
self._encoder_buffer.write(varint.encode(len(data)))
# Don't initialize the encoder before the first call to encode(), since
# it writes the gzip header immediately and we need to insert the
# message length prior to that happening.
if self._encoder is None:
self._encoder = GzipFile(fileobj=self._encoder_buffer,
mode='wb',
compresslevel=self._compression_level)
self._encoder.write(data)
self._encoder.flush()
output = self._encoder_buffer.getvalue()
reset_buffer(self._encoder_buffer)
return output
def decode(self, data):
# Decode the varuint prefix off the data first, then smash the remaining
# data into the decode buffer and reset it to read any previous tail.
prefix_stream = io.BytesIO(data)
decoded_bytes = varint.decode_stream(prefix_stream)
self._decoder_buffer.write(data[prefix_stream.tell():])
self._decoder_buffer.seek(0)
decoded_data = self._decoder.decompress(
self._decoder_buffer.getbuffer(), decoded_bytes)
reset_buffer(self._decoder_buffer, self._decoder.unconsumed_tail)
return decoded_data.decode('utf-8')
def create_metadata_record_bytes(
url='http://example.com/',
content_type='image/png',
date='2016-08-03T10:49:41Z',
content=b'',
include_block_digest=True):
"""Build WARC metadata record bits."""
headers = {
WarcRecord.TYPE: WarcRecord.METADATA,
WarcRecord.URL: url.encode('utf-8'),
WarcRecord.CONTENT_TYPE: content_type.encode('utf-8'),
WarcRecord.DATE: date.encode('utf-8')
}
if include_block_digest:
hasher = hashlib.sha1(content)
block_digest = base64.b32encode(hasher.digest())
headers[WarcRecord.BLOCK_DIGEST] = b'sha1:' + block_digest
# XXX - I wish I could use WarcRecord. Current implementation of
# WarcRecord.write_to() ignores Warc-Block-Digest passed and writes out
# hex-encoded SHA256 calculated from the content.
out = io.BytesIO()
if False:
rec = WarcRecord(
headers=headers.items(),
content=(content_type.encode('utf-8'), content)
)
out = io.BytesIO()
rec.write_to(out, gzip=True)
return out.getvalue()
else:
z = GzipFile(fileobj=out, mode='wb')
z.write(b'WARC/1.0\r\n')
for k, v in headers.items():
z.write(b''.join((k, b': ', v, b'\r\n')))
z.write('Content-Length: {}\r\n'.format(len(content)).encode('ascii'))
z.write(b'\r\n')
z.write(content)
z.write(b'\r\n\r\n')
z.flush()
z.close()
return out.getvalue()
def write_to(self, out, newline=b'\x0D\x0A', gzip=False):
if self.content_file is not None:
if not self._content_file_valid:
raise Exception('cannot write record because content_file has already been used')
if gzip:
if hasattr(out, 'mode'):
out = GzipFile(fileobj=out)
else:
out = GzipFile(fileobj=out, mode='ab')
self._write_to(out, newline)
if gzip:
out.flush()
out.close()
if self.content_file is not None:
self._content_file_valid = False
def write_to(self, out, newline=b'\x0D\x0A', gzip=False):
if self.content_file is not None:
if not self._content_file_valid:
raise Exception(
'cannot write record because content_file has already been used'
)
if gzip:
if hasattr(out, 'mode'):
out = GzipFile(fileobj=out)
else:
out = GzipFile(fileobj=out, mode='ab')
self._write_to(out, newline)
if gzip:
out.flush()
out.close()
if self.content_file is not None:
self._content_file_valid = False
class _GzipEncoder(object):
def __init__(self, request, preamble):
preamble.headers['Vary'] = preamble.headers.get('Vary', '') + ',Accept-Encoding'
self._accepted = 'gzip' in request.headers.get('Accept-Encoding', '').replace(' ','').split(',')
if self._accepted:
self._value = BytesIO()
self._file = GzipFile(mode='wb', fileobj=self._value)
preamble.headers['Content-Encoding'] = preamble.headers.get('Content-Encoding', '') + ',gzip'
def encode(self, data):
if self._accepted:
self._file.write(data)
self._file.flush()
data = self._value.getvalue()
self._value.truncate(0)
self._value.seek(0)
return data
def finish(self, data):
if self._accepted:
data = self.encode(data)
self._file.close()
return data
class Gzipper(object):
"""
Text stream compressor.
"""
def __init__(self, filename=None, compress=False):
self.compress = compress
if not self.compress:
self.buffer = ''
else:
self.io = StringIO()
self.zipfile = GzipFile(filename, mode='wb', fileobj=self.io)
def read(self):
if not self.compress:
result = self.buffer
self.buffer = ''
return result
self.zipfile.flush()
self.io.seek(0)
line = self.io.read()
self.io.seek(0)
self.io.truncate()
return line
def write(self, l):
if not self.compress:
self.buffer += l
else:
self.zipfile.write(l)
def close(self):
if not self.compress:
return self.buffer
self.zipfile.close()
self.io.seek(0)
return self.io.read()
def compress_sequence(sequence):
avg_block_size = 1.0 / AVERAGE_SPAN_BETWEEN_FLUSHES
buf = StreamingBuffer()
zfile = GzipFile(mode='wb', compresslevel=6, fileobj=buf)
# Output headers...
yield buf.read()
flushes_remaining = FLUSH_LIMIT
rnd = None
count = None
rnd = None
for item in sequence:
if rnd is None:
rnd = Random(hash(item))
count = int(rnd.expovariate(avg_block_size))
chunking_buf = BytesIO(item)
chunk = chunking_buf.read(count)
while chunk:
if count is not None:
count -= len(chunk)
zfile.write(chunk)
if count <= 0:
flushes_remaining -= 1
zfile.flush()
yield buf.read()
if flushes_remaining:
count = int(rnd.expovariate(avg_block_size))
else:
count = None
if count is None:
chunk = chunking_buf.read()
else:
chunk = chunking_buf.read(count)
zfile.flush()
yield buf.read()
if chunk is None:
break
for item in sequence:
zfile.write(chunking_buf.read())
zfile.flush()
yield buf.read()
zfile.close()
yield buf.read()
def compress_sequence(sequence):
avg_block_size = 1.0 / AVERAGE_SPAN_BETWEEN_FLUSHES
buf = StreamingBuffer()
zfile = GzipFile(mode='wb', compresslevel=6, fileobj=buf)
# Output headers...
yield buf.read()
flushes_remaining = FLUSH_LIMIT
rnd = None
count = None
rnd = None
for item in sequence:
if rnd is None:
rnd = Random(0)
count = int(rnd.expovariate(avg_block_size))
chunking_buf = BytesIO(item)
chunk = chunking_buf.read(count)
while chunk:
if count is not None:
count -= len(chunk)
zfile.write(chunk)
if count <= 0:
flushes_remaining -= 1
zfile.flush()
yield buf.read()
if flushes_remaining:
count = int(rnd.expovariate(avg_block_size))
else:
count = None
if count is None:
chunk = chunking_buf.read()
else:
chunk = chunking_buf.read(count)
zfile.flush()
yield buf.read()
if chunk is None:
break
for item in sequence:
zfile.write(chunking_buf.read())
zfile.flush()
yield buf.read()
zfile.close()
yield buf.read()
class GZipPipe(StringIO) :
"""This class implements a compression pipe suitable for asynchronous
process.
Only one buffer of data is read/compressed at a time.
The process doesn't read the whole file at once : This improve performance
and prevent hight memory consumption for big files."""
# Size of the internal buffer
CHUNCK_SIZE = 1024
def __init__(self, source = None, name = "data") :
"""Constructor
@param source Source data to compress (as a stream/File/Buffer - anything with a read() method)
@param name Name of the data within the zip file"""
# Source file
self.source = source
# OEF reached for source ?
self.source_eof = False
# Buffer
self.buffer = ""
StringIO.__init__(self)
# Inherited constructor
# Init ZipFile that writes to us (the StringIO buffer)
self.zipfile = GzipFile(name, 'wb', 9, self)
def write(self, data) :
"""The write mzthod shouldn't be called from outside.
A GZipFile was created with this current object as a output buffer anbd it
fills it whenever we write to it (calling the read method of this object will do it for you)
"""
self.buffer += data
def read(self, size = -1) :
"""Calling read() on a zip pipe will suck data from the source stream.
@param size Maximum size to read - Read whole compressed file if not specified.
@return Compressed data"""
# Feed the zipped buffer by writing source data to the zip stream
while ((len(self.buffer) < size) or (size == -1)) and not self.source_eof :
# No source given in input
if self.source == None: break
# Get a chunk of source data
chunk = self.source.read(GZipPipe.CHUNCK_SIZE)
# Feed the source zip file (that fills the compressed buffer)
self.zipfile.write(chunk)
# End of source file ?
if (len(chunk) < GZipPipe.CHUNCK_SIZE) :
self.source_eof = True
self.zipfile.flush()
self.zipfile.close()
break
# We have enough data in the buffer (or source file is EOF): Give it to the output
if size == 0:
result = ""
if size >= 1 :
result = self.buffer[0:size]
self.buffer = self.buffer[size:]
else : # size < 0 : All requested
result = self.buffer
self.buffer = ""
return result
def fork_map(
func,
arg_list: typing.Iterable,
workers: int = 10,
tmp_dir=None,
gzip=False,
gzip_level=4,
) -> typing.Generator[None, typing.Any, None]:
result_files = [
NamedTemporaryFile(
mode="wb+",
delete=False,
dir=tmp_dir,
prefix="fork-map-",
suffix=".results.gz" if gzip else ".results",
) for _ in range(workers)
]
if not isinstance(arg_list, list):
arg_list = list(arg_list)
children = set()
freeze()
for i in range(workers):
pid = os.fork()
if pid:
# Is parent
children.add(pid)
continue
# Is child
if gzip:
result_file = GzipFile(
fileobj=result_files[i],
mode="wb",
compresslevel=gzip_level,
)
else:
result_file = result_files[i]
for task in arg_list[i::workers]:
try:
res = pickle.dumps((func(task), False))
except Exception as e:
res = pickle.dumps((e, True))
result_file.write(HEADER.pack(len(res)))
result_file.write(res)
result_file.flush()
return exit(0)
unfreeze()
while children:
pid, code = os.wait()
if code:
raise RuntimeError(
"Child process %d exited with code %r",
pid,
code,
)
children.remove(pid)
for i in range(workers):
result_files[i].seek(0)
if gzip:
result_read_files = [
GzipFile(fileobj=fp, mode="rb") for fp in result_files
]
else:
result_read_files = result_files
try:
for i in range(len(arg_list)):
fp = result_read_files[i % workers]
hdr = fp.read(HEADER.size)
size = HEADER.unpack(hdr)[0]
res, exc = pickle.loads(fp.read(size))
if exc:
raise exc
yield res
finally:
for fp in result_files:
os.remove(fp.name)
class NBTFile(TAG_Compound):
"""Represents an NBT file object"""
def __init__(self, filename=None, mode=None, buffer=None, fileobj=None):
super(NBTFile,self).__init__()
self.__class__.__name__ = "TAG_Compound"
self.filename = filename
self.type = TAG_Byte(self.id)
#make a file object
if filename:
self.file = GzipFile(filename, mode)
elif buffer:
self.file = buffer
elif fileobj:
self.file = GzipFile(fileobj=fileobj)
else:
self.file = None
#parse the file given intitially
if self.file:
self.parse_file()
if filename and 'close' in dir(self.file):
self.file.close()
self.file = None
def parse_file(self, filename=None, buffer=None, fileobj=None):
if filename:
self.file = GzipFile(filename, 'rb')
elif buffer:
self.file = buffer
elif fileobj:
self.file = GzipFile(fileobj=fileobj)
if self.file:
try:
type = TAG_Byte(buffer=self.file)
if type.value == self.id:
name = TAG_String(buffer=self.file).value
self._parse_buffer(self.file)
self.name = name
self.file.close()
else:
raise MalformedFileError("First record is not a Compound Tag")
except StructError as e:
raise MalformedFileError("Partial File Parse: file possibly truncated.")
else: ValueError("need a file!")
def write_file(self, filename=None, buffer=None, fileobj=None):
if buffer:
self.file = buffer
elif filename:
self.filename = filename
self.file = GzipFile(filename, "wb")
elif fileobj:
self.file = GzipFile(fileobj=fileobj)
elif self.filename:
self.file = GzipFile(self.filename, "wb")
elif not self.file:
raise ValueError("Need to specify either a filename or a file")
#Render tree to file
TAG_Byte(self.id)._render_buffer(self.file)
TAG_String(self.name)._render_buffer(self.file)
self._render_buffer(self.file)
#make sure the file is complete
if 'flush' in dir(self.file):
self.file.flush()
if self.filename and 'close' in dir(self.file):
self.file.close()
def flush(self, lib_mode=FLUSH):
self._buffer.flush()
GzipFile.flush(self, lib_mode)
class NBTFile(TAG_Compound):
"""Represent an NBT file object."""
def __init__(self, filename=None, buffer=None, fileobj=None):
"""
Create a new NBTFile object.
Specify either a filename, file object or data buffer.
If filename of file object is specified, data should be GZip-compressed.
If a data buffer is specified, it is assumed to be uncompressed.
If filename is specified, the file is closed after reading and writing.
If file object is specified, the caller is responsible for closing the
file.
"""
super(NBTFile, self).__init__()
self.filename = filename
self.type = TAG_Byte(self.id)
closefile = True
# make a file object
if filename:
self.filename = filename
self.file = GzipFile(filename, 'rb')
elif buffer:
if hasattr(buffer, 'name'):
self.filename = buffer.name
self.file = buffer
closefile = False
elif fileobj:
if hasattr(fileobj, 'name'):
self.filename = fileobj.name
self.file = GzipFile(fileobj=fileobj)
else:
self.file = None
closefile = False
# parse the file given initially
if self.file:
self.parse_file()
if closefile:
# Note: GzipFile().close() does NOT close the fileobj,
# So we are still responsible for closing that.
try:
self.file.close()
except (AttributeError, IOError):
pass
self.file = None
def parse_file(self, filename=None, buffer=None, fileobj=None):
"""Completely parse a file, extracting all tags."""
if filename:
self.file = GzipFile(filename, 'rb')
elif buffer:
if hasattr(buffer, 'name'):
self.filename = buffer.name
self.file = buffer
elif fileobj:
if hasattr(fileobj, 'name'):
self.filename = fileobj.name
self.file = GzipFile(fileobj=fileobj)
if self.file:
try:
type = TAG_Byte(buffer=self.file)
if type.value == self.id:
name = TAG_String(buffer=self.file).value
self._parse_buffer(self.file)
self.name = name
self.file.close()
else:
raise MalformedFileError(
"First record is not a Compound Tag")
except StructError as e:
raise MalformedFileError(
"Partial File Parse: file possibly truncated.")
else:
raise ValueError(
"NBTFile.parse_file(): Need to specify either a "
"filename or a file object"
)
def write_file(self, filename=None, buffer=None, fileobj=None):
"""Write this NBT file to a file."""
closefile = True
if buffer:
self.filename = None
self.file = buffer
closefile = False
elif filename:
self.filename = filename
self.file = GzipFile(filename, "wb")
elif fileobj:
self.filename = None
self.file = GzipFile(fileobj=fileobj, mode="wb")
elif self.filename:
self.file = GzipFile(self.filename, "wb")
elif not self.file:
raise ValueError(
"NBTFile.write_file(): Need to specify either a "
"filename or a file object"
)
# Render tree to file
TAG_Byte(self.id)._render_buffer(self.file)
TAG_String(self.name)._render_buffer(self.file)
self._render_buffer(self.file)
# make sure the file is complete
try:
self.file.flush()
except (AttributeError, IOError):
pass
if closefile:
try:
self.file.close()
except (AttributeError, IOError):
pass
def __repr__(self):
"""
Return a string (ascii formated for Python 2, unicode
for Python 3) describing the class, name and id for
debugging purposes.
"""
if self.filename:
return "<%s(%r) with %s(%r) at 0x%x>" % (
self.__class__.__name__, self.filename,
TAG_Compound.__name__, self.name, id(self)
)
else:
return "<%s with %s(%r) at 0x%x>" % (
self.__class__.__name__, TAG_Compound.__name__,
self.name, id(self)
)
def flush(self, lib_mode=FLUSH):
self._buffer.flush()
GzipFile.flush(self, lib_mode)
class NBTFile(TAG_Compound):
"""Represent an NBT file object."""
def __init__(self, filename=None, buffer=None, fileobj=None):
super(NBTFile, self).__init__()
self.filename = filename
self.type = TAG_Byte(self.id)
#make a file object
if filename:
self.file = GzipFile(filename, 'rb')
elif buffer:
self.file = buffer
elif fileobj:
self.file = GzipFile(fileobj=fileobj)
else:
self.file = None
#parse the file given intitially
if self.file:
self.parse_file()
if self.filename and 'close' in dir(self.file):
self.file.close()
self.file = None
def parse_file(self, filename=None, buffer=None, fileobj=None):
"""Completely parse a file, extracting all tags."""
if filename:
self.file = GzipFile(filename, 'rb')
elif buffer:
self.file = buffer
elif fileobj:
self.file = GzipFile(fileobj=fileobj)
if self.file:
try:
type = TAG_Byte(buffer=self.file)
if type.value == self.id:
name = TAG_String(buffer=self.file).value
self._parse_buffer(self.file)
self.name = name
self.file.close()
else:
raise MalformedFileError("First record is not a Compound Tag")
except StructError as e:
raise MalformedFileError("Partial File Parse: file possibly truncated.")
else: ValueError("need a file!")
def write_file(self, filename=None, buffer=None, fileobj=None):
"""Write this NBT file to a file."""
if buffer:
self.filename = None
self.file = buffer
elif filename:
self.filename = filename
self.file = GzipFile(filename, "wb")
elif fileobj:
self.filename = None
self.file = GzipFile(fileobj=fileobj, mode="wb")
elif self.filename:
self.file = GzipFile(self.filename, "wb")
elif not self.file:
raise ValueError("Need to specify either a filename or a file")
#Render tree to file
TAG_Byte(self.id)._render_buffer(self.file)
TAG_String(self.name)._render_buffer(self.file)
self._render_buffer(self.file)
#make sure the file is complete
if 'flush' in dir(self.file):
self.file.flush()
if self.filename and 'close' in dir(self.file):
self.file.close()
def __repr__(self):
"""
Return a string (ascii formated for Python 2, unicode
for Python 3) describing the class, name and id for
debugging purposes.
"""
if self.filename:
return "<%s(%r) with %s(%r) at 0x%x>" % (self.__class__.__name__, self.filename, \
TAG_Compound.__name__, self.name, id(self))
else:
return "<%s with %s(%r) at 0x%x>" % (self.__class__.__name__, \
TAG_Compound.__name__, self.name, id(self))
class NBTFile(Tag_Compound):
"""Represent an NBT file object."""
def __init__(self, filename=None, buffer=None, fileobj=None):
"""
Create a new NBTFile object.
Specify either a filename, file object or data buffer.
If filename of file object is specified, data should be GZip-compressed.
If a data buffer is specified, it is assumed to be uncompressed.
If filename is specified, the file is closed after reading and writing.
If file object is specified, the caller is responsible for closing the
file.
"""
super(NBTFile, self).__init__()
self.filename = filename
self.type = TAG_Byte(self.id)
closefile = True
# make a file object
if filename:
self.filename = filename
self.file = GzipFile(filename, 'rb')
elif buffer:
if hasattr(buffer, 'name'):
self.filename = buffer.name
self.file = buffer
closefile = False
elif fileobj:
if hasattr(fileobj, 'name'):
self.filename = fileobj.name
self.file = GzipFile(fileobj=fileobj)
else:
self.file = None
closefile = False
# parse the file given initially
if self.file:
self.parse_file()
if closefile:
# Note: GzipFile().close() does NOT close the fileobj,
# So we are still responsible for closing that.
try:
self.file.close()
except (AttributeError, IOError):
pass
self.file = None
def parse_file(self, filename=None, buffer=None, fileobj=None):
"""Completely parse a file, extracting all tags."""
if filename:
self.file = GzipFile(filename, 'rb')
elif buffer:
if hasattr(buffer, 'name'):
self.filename = buffer.name
self.file = buffer
elif fileobj:
if hasattr(fileobj, 'name'):
self.filename = fileobj.name
self.file = GzipFile(fileobj=fileobj)
if self.file:
try:
type = TAG_Byte(buffer=self.file)
if type.value == self.id:
name = Tag_String(buffer=self.file).value
self._parse_buffer(self.file)
self.name = name
self.file.close()
else:
raise MalformedFileError(
"First record is not a Compound Tag")
except StructError as e:
raise MalformedFileError(
"Partial File Parse: file possibly truncated.")
else:
raise ValueError("NBTFile.parse_file(): Need to specify either a "
"filename or a file object")
def write_file(self, filename=None, buffer=None, fileobj=None):
"""Write this NBT file to a file."""
closefile = True
if buffer:
self.filename = None
self.file = buffer
closefile = False
elif filename:
self.filename = filename
self.file = GzipFile(filename, "wb")
elif fileobj:
self.filename = None
self.file = GzipFile(fileobj=fileobj, mode="wb")
elif self.filename:
self.file = GzipFile(self.filename, "wb")
elif not self.file:
raise ValueError("NBTFile.write_file(): Need to specify either a "
"filename or a file object")
# Render tree to file
TAG_Byte(self.id)._render_buffer(self.file)
Tag_String(self.name)._render_buffer(self.file)
self._render_buffer(self.file)
# make sure the file is complete
try:
self.file.flush()
except (AttributeError, IOError):
pass
if closefile:
try:
self.file.close()
except (AttributeError, IOError):
pass
def __repr__(self):
if self.filename:
return "<{}({!r}) with {}({!r}) at {:#x}>".format(
self.__class__.__name__, self.filename, Tag_Compound.__name__,
self.name, id(self))
else:
return "<{} with {}({!r}) at {:#x}>".format(
self.__class__.__name__,
Tag_Compound.__name__,
self.name,
id(self),
)
class GZipPipe(StringIO):
"""This class implements a compression pipe suitable for asynchronous
process.
Only one buffer of data is read/compressed at a time.
The process doesn't read the whole file at once : This improves performance
and prevent hight memory consumption for big files."""
# Size of the internal buffer
CHUNCK_SIZE = 1024
def __init__(self, source=None, name="data"):
"""Constructor
@param source Source data to compress (as a stream/File/Buffer - anything with a read() method)
@param name Name of the data within the zip file"""
# Source file
self.source = source
# OEF reached for source ?
self.source_eof = False
# Buffer
self.buffer = ""
StringIO.__init__(self)
# Inherited constructor
# Init ZipFile that writes to us (the StringIO buffer)
self.zipfile = GzipFile(name, 'wb', 9, self)
def write(self, data):
"""The write mzthod shouldn't be called from outside.
A GZipFile was created with this current object as a output buffer anbd it
fills it whenever we write to it (calling the read method of this object will do it for you)
"""
self.buffer += data
def read(self, size=-1):
"""Calling read() on a zip pipe will suck data from the source stream.
@param size Maximum size to read - Read whole compressed file if not specified.
@return Compressed data"""
# Feed the zipped buffer by writing source data to the zip stream
while ((len(self.buffer) < size) or
(size == -1)) and not self.source_eof:
# No source given in input
if self.source == None: break
# Get a chunk of source data
chunk = self.source.read(GZipPipe.CHUNCK_SIZE)
# Feed the source zip file (that fills the compressed buffer)
self.zipfile.write(chunk)
# End of source file ?
if (len(chunk) < GZipPipe.CHUNCK_SIZE):
self.source_eof = True
self.zipfile.flush()
self.zipfile.close()
break
# We have enough data in the buffer (or source file is EOF): Give it to the output
if size == 0:
result = ""
if size >= 1:
result = self.buffer[0:size]
self.buffer = self.buffer[size:]
else: # size < 0 : All requested
result = self.buffer
self.buffer = ""
return result
class NBTFile(TAG_Compound):
"""Represent an NBT file object."""
def __init__(self, filename=None, buffer=None, fileobj=None):
super(NBTFile, self).__init__()
self.filename = filename
self.type = TAG_Byte(self.tid)
closefile = True
# make a file object
if filename:
self.file = GzipFile(filename, 'rb')
elif buffer:
if hasattr(buffer, 'name'):
self.filename = buffer.name
self.file = buffer
closefile = False
elif fileobj:
if hasattr(fileobj, 'name'):
self.filename = fileobj.name
self.file = GzipFile(fileobj=fileobj)
else:
self.file = None
closefile = False
# parse the file given initially
if self.file:
self.parse_file()
if closefile:
# Note: GzipFile().close() does NOT close the fileobj,
# So the caller is still responsible for closing that.
try:
self.file.close()
except (AttributeError, IOError):
pass
self.file = None
def parse_file(self, filename=None, buffer=None, fileobj=None):
"""Completely parse a file, extracting all tags."""
if filename:
self.file = GzipFile(filename, 'rb')
elif buffer:
if hasattr(buffer, 'name'):
self.filename = buffer.name
self.file = buffer
elif fileobj:
if hasattr(fileobj, 'name'):
self.filename = fileobj.name
self.file = GzipFile(fileobj=fileobj)
if self.file:
try:
type = TAG_Byte(buffer=self.file)
if type.value == self.tid:
name = TAG_String(buffer=self.file).value
self._parse_buffer(self.file)
self.name = name
self.file.close()
else:
raise MalformedFileError(
"First record is not a Compound Tag")
except StructError as e:
raise MalformedFileError(
"Partial File Parse: file possibly truncated.")
else:
raise ValueError(
"NBTFile.parse_file(): Need to specify either a filename or a file object")
def write_file(self, filename=None, buffer=None, fileobj=None):
"""Write this NBT file to a file."""
closefile = True
if buffer:
self.filename = None
self.file = buffer
closefile = False
elif filename:
self.filename = filename
self.file = GzipFile(filename, "wb")
elif fileobj:
self.filename = None
self.file = GzipFile(fileobj=fileobj, mode="wb")
elif self.filename:
self.file = GzipFile(self.filename, "wb")
elif not self.file:
raise ValueError("NBTFile.write_file(): Need to specify either a filename or a file object")
# Render tree to file
TAG_Byte(self.tid)._render_buffer(self.file)
TAG_String(self.name)._render_buffer(self.file)
self._render_buffer(self.file)
# make sure the file is complete
try:
self.file.flush()
except (AttributeError, IOError):
pass
if closefile:
try:
self.file.close()
except (AttributeError, IOError):
pass
def __repr__(self):
"""
Return a string (ascii formated for Python 2, unicode
for Python 3) describing the class, name and id for
debugging purposes.
"""
if self.filename:
return "<%s(%r) with %s(%r) at 0x%x>" % \
(self.__class__.__name__, self.filename, TAG_Compound.__name__, self.name, id(self))
else:
return "<%s with %s(%r) at 0x%x>" % \
(self.__class__.__name__, TAG_Compound.__name__, self.name, id(self))
class NBTFile(TAG_Compound):
"""Represent an NBT file object."""
def __init__(self, filename=None, buffer=None, fileobj=None):
super(NBTFile, self).__init__()
self.filename = filename
self.type = TAG_Byte(self.id)
#make a file object
if filename:
self.file = GzipFile(filename, 'rb')
elif buffer:
self.file = buffer
elif fileobj:
self.file = GzipFile(fileobj=fileobj)
else:
self.file = None
#parse the file given intitially
if self.file:
self.parse_file()
if self.filename and 'close' in dir(self.file):
self.file.close()
self.file = None
def parse_file(self, filename=None, buffer=None, fileobj=None):
"""Completely parse a file, extracting all tags."""
if filename:
self.file = GzipFile(filename, 'rb')
elif buffer:
self.file = buffer
elif fileobj:
self.file = GzipFile(fileobj=fileobj)
if self.file:
try:
type = TAG_Byte(buffer=self.file)
if type.value == self.id:
name = TAG_String(buffer=self.file).value
self._parse_buffer(self.file)
self.name = name
self.file.close()
else:
raise MalformedFileError(
"First record is not a Compound Tag")
except StructError as e:
raise MalformedFileError(
"Partial File Parse: file possibly truncated.")
else:
ValueError("need a file!")
def write_file(self, filename=None, buffer=None, fileobj=None):
"""Write this NBT file to a file."""
if buffer:
self.filename = None
self.file = buffer
elif filename:
self.filename = filename
self.file = GzipFile(filename, "wb")
elif fileobj:
self.filename = None
self.file = GzipFile(fileobj=fileobj, mode="wb")
elif self.filename:
self.file = GzipFile(self.filename, "wb")
elif not self.file:
raise ValueError("Need to specify either a filename or a file")
#Render tree to file
TAG_Byte(self.id)._render_buffer(self.file)
TAG_String(self.name)._render_buffer(self.file)
self._render_buffer(self.file)
#make sure the file is complete
if 'flush' in dir(self.file):
self.file.flush()
if self.filename and 'close' in dir(self.file):
self.file.close()
def __repr__(self):
"""
Return a string (ascii formated for Python 2, unicode
for Python 3) describing the class, name and id for
debugging purposes.
"""
if self.filename:
return "<%s(%r) with %s(%r) at 0x%x>" % (self.__class__.__name__, self.filename, \
TAG_Compound.__name__, self.name, id(self))
else:
return "<%s with %s(%r) at 0x%x>" % (self.__class__.__name__, \
TAG_Compound.__name__, self.name, id(self))
