def _writeConstantStream(constants_reader):
result = BytesIO()
count = 0
while 1:
try:
constant_value = constants_reader.readConstantValue()
except EOFError:
break
old_size = result.tell()
_writeConstantValue(result, constant_value)
if not datacomposer_logger.is_quiet:
new_size = result.tell()
result.seek(old_size)
type_char = result.read(1)
result.seek(new_size)
datacomposer_logger.info(
"Size of constant %r is %d with type %r" %
(constant_value, new_size - old_size, type_char))
count += 1
# Dirty end of things marker.
result.write(b".")
return count, result.getvalue()
def _writeConstantStream(constants_reader):
result = BytesIO()
# We are a singleton, pylint: disable=global-statement
global _last_written
_last_written = None
count = 0
while 1:
try:
constant_value = constants_reader.readConstantValue()
except EOFError:
break
old_size = result.tell()
_writeConstantValue(result, constant_value)
if not datacomposer_logger.is_quiet:
new_size = result.tell()
result.seek(old_size)
type_char = result.read(1)
result.seek(new_size)
datacomposer_logger.info(
"Size of constant %r is %d with type %r" %
(constant_value, new_size - old_size, type_char))
count += 1
# Dirty end of things marker that would trigger an assertion in the decoder.
# TODO: Debug mode only?
result.write(b".")
return count, struct.pack("h", count) + result.getvalue()
def _writeConstantsBlob(output_filename, desc):
global crc32 # singleton, pylint: disable=global-statement
with open(output_filename, "w+b") as output:
output.write(b"\0" * 8)
def write(data):
global crc32 # singleton, pylint: disable=global-statement
output.write(data)
crc32 = binascii.crc32(data, crc32)
for name, part in desc:
write(name + b"\0")
write(struct.pack("I", len(part)))
write(part)
data_size = output.tell() - 8
if str is bytes:
# Python2 is doing signed CRC32, but we want unsigned.
crc32 %= 1 << 32
output.seek(0)
output.write(struct.pack("II", crc32, data_size))
assert output.tell() == 8
datacomposer_logger.info(
"Total constants blob size without header %d." % data_size)
datacomposer_logger.info("Total constants blob CRC32 is %d." % crc32)
def main():
datacomposer_logger.is_quiet = (
os.environ.get("NUITKA_DATACOMPOSER_VERBOSE", "0") != "1"
)
# Internal tool, most simple command line handling. This is the build directory
# where main Nuitka put the .const files.
build_dir = sys.argv[1]
output_filename = sys.argv[2]
const_files = scanConstFiles(build_dir)
total = 0
desc = []
names = set()
for fullpath, filename in const_files:
datacomposer_logger.info("Working on constant file %r." % filename)
with open(fullpath, "rb") as const_file:
constants_reader = ConstantStreamReader(const_file)
count, part = _writeConstantStream(constants_reader)
total += count
name = deriveModuleConstantsBlobName(filename)
# Make sure that is not repeated.
assert name not in names, name
names.add(name)
datacomposer_logger.info(
"Storing %r chunk with %s values size %r." % (name, count, len(part))
)
if str is not bytes:
# Encoding needs to match generated source code output.
name = name.encode("latin1")
desc.append((name, part))
datacomposer_logger.info("Total amount of constants is %d." % total)
_writeConstantsBlob(output_filename=output_filename, desc=desc)
sys.exit(0)
def main():
global crc32 # singleton, pylint: disable=global-statement
datacomposer_logger.is_quiet = (os.environ.get(
"NUITKA_DATACOMPOSER_VERBOSE", "0") != "1")
# Internal tool, most simple command line handling. This is the build directory
# where main Nuitka put the .const files.
build_dir = sys.argv[1]
output_filename = sys.argv[2]
const_files = scanConstFiles(build_dir)
total = 0
desc = []
names = set()
for fullpath, filename in const_files:
datacomposer_logger.info("Working on constant file %r." % filename)
constants_reader = ConstantStreamReader(fullpath)
count, part = _writeConstantStream(constants_reader)
total += count
name = deriveModuleConstantsBlobName(filename)
# Make sure that is not repeated.
assert name not in names, name
names.add(name)
datacomposer_logger.info("Storing %r chunk with %s values size %r." %
(name, count, len(part)))
if str is not bytes:
# Encoding needs to match generated source code output.
name = name.encode("latin1")
desc.append((name, part))
datacomposer_logger.info("Total amount of constants is %d." % total)
with open(output_filename, "w+b") as output:
output.write(b"\0" * 8)
def write(data):
global crc32 # singleton, pylint: disable=global-statement
output.write(data)
crc32 = binascii.crc32(data, crc32)
for name, part in desc:
write(name + b"\0")
write(struct.pack("I", len(part)))
write(part)
data_size = output.tell() - 8
if str is bytes:
# Python2 is doing signed CRC32, but we want unsigned.
crc32 %= 1 << 32
output.seek(0)
output.write(struct.pack("II", crc32, data_size))
assert output.tell() == 8
datacomposer_logger.info(
"Total constants blob size without header %d." % data_size)
datacomposer_logger.info("Total constants blob CRC32 is %d." % crc32)
sys.exit(0)