def link_binary(self, objcopy, src, dst):
"""Link a binary file with no bells and whistles."""
ld_target = dst
cmd = [self.__command, "--entry=" + str(PlatformVar("entry"))] + listify(src) + self.__linker_script + self.__linker_flags_extra
# Use objcopy if it was given.
if objcopy:
(dst_base, dst_ext) = os.path.splitext(dst)
dst_bin = dst_base + ".out"
objcopy_cmd = [objcopy, "--output-target=binary", dst_bin, dst]
ld_target = dst_bin
# Otherwise link directly into binary.
else:
cmd += ["--oformat=binary"]
cmd += ["-o", ld_target]
# Run linker command.
(so, se) = run_command(cmd)
if 0 < len(se) and is_verbose():
print(se)
# Only run objcopy commad if it was required.
if objcopy:
(so_add, se) = run_command(objcopy_cmd)
if 0 < len(se) and is_verbose():
print(se)
so += so_add
return so
def link_binary(self, objcopy, src, dst):
"""Link a binary file with no bells and whistles."""
ld_target = dst
cmd = [
self.__command, "--entry=" + str(PlatformVar("entry"))
] + listify(src) + self.__linker_script + self.__linker_flags_extra
# Use objcopy if it was given.
if objcopy:
(dst_base, dst_ext) = os.path.splitext(dst)
dst_bin = dst_base + ".out"
objcopy_cmd = [objcopy, "--output-target=binary", dst_bin, dst]
ld_target = dst_bin
# Otherwise link directly into binary.
else:
cmd += ["--oformat=binary"]
cmd += ["-o", ld_target]
# Run linker command.
(so, se) = run_command(cmd)
if 0 < len(se) and is_verbose():
print(se)
# Only run objcopy commad if it was required.
if objcopy:
(so_add, se) = run_command(objcopy_cmd)
if 0 < len(se) and is_verbose():
print(se)
so += so_add
return so
def generate_linker_script(self, dst, modify_start=False):
"""Get linker script from linker, improve it, write improved linker script to given file."""
(so, se) = run_command([self.__command, "--verbose"] +
self.__linker_flags_extra)
if 0 < len(se) and is_verbose():
print(se)
# Linker script is the block of code between lines of multiple '=':s.
match = re.match(r'.*\n=+\s*\n(.*)\n=+\s*\n.*', so, re.DOTALL)
if not match:
raise RuntimeError("could not extract script from linker output")
ld_script = match.group(1)
# Remove unwanted symbol definitions one at a time.
unwanted_symbols = [
"__bss_end__", "__bss_start__", "__end__", "__bss_start",
"_bss_end__", "_edata", "_end"
]
for ii in unwanted_symbols:
ld_script = re.sub(r'\n([ \f\r\t\v]+)(%s)(\s*=[^\n]+)\n' % (ii),
r'\n\1/*\2\3*/\n', ld_script, re.MULTILINE)
ld_script = re.sub(
r'SEGMENT_START\s*\(\s*(\S+)\s*,\s*\d*x?\d+\s*\)',
r'SEGMENT_START(\1, %s)' % (str(PlatformVar("entry"))), ld_script,
re.MULTILINE)
if modify_start:
ld_script = re.sub(
r'(SEGMENT_START.*\S)\s*\+\s*SIZEOF_HEADERS\s*;', r'\1;',
ld_script, re.MULTILINE)
fd = open(dst, "w")
fd.write(ld_script)
fd.close()
if is_verbose():
print("Wrote linker script '%s'." % (dst))
return ld_script
def crunch_entry_push(self, op):
"""Crunch amd64/ia32 push directives from given line listing."""
lst = self.want_label(op)
if not lst:
return
ii = lst[0] + 1
jj = ii
stack_decrement = 0
reinstated_lines = []
while True:
current_line = self.__content[jj]
match = re.match(r'\s*(push\w).*%(\w+)', current_line,
re.IGNORECASE)
if match:
stack_decrement += get_push_size(match.group(1))
jj += 1
continue
# Preserve comment lines as they are.
match = re.match(r'^\s*[#;].*', current_line, re.IGNORECASE)
if match:
reinstated_lines += [current_line]
jj += 1
continue
# Some types of lines can be in the middle of pushing.
if is_reinstate_line(current_line):
reinstated_lines += [current_line]
jj += 1
continue
# Stop at stack decrement.
match = re.match(r'\s*sub.*\s+[^\d]*(\d+),\s*%(rsp|esp)',
current_line, re.IGNORECASE)
if match:
# Align to 16 bytes if necessary.
if osname_is_linux() and osarch_is_64_bit():
if osarch_is_amd64():
# Just ignore increment, there's probably enough stack.
self.__content[jj] = re.sub(
r'subq(\s*).*',
r'andq\g<1>$0xFFFFFFFFFFFFFFF0, %rsp',
current_line)
else:
raise RuntimeError(
"no stack alignment instruction for current architecture"
)
else:
total_decrement = int(match.group(1)) + stack_decrement
self.__content[jj] = re.sub(r'\d+', str(total_decrement),
current_line)
break
# Do nothing if suspicious instruction is found.
if is_verbose():
print(
"Unknown header instruction found, aborting erase: '%s'" %
(current_line.strip()))
break
if is_verbose():
print("Erasing function header from '%s': %i lines" %
(op, jj - ii - len(reinstated_lines)))
self.__content[ii:jj] = reinstated_lines
def link(self, src, dst, extra_args=[]):
"""Link a file."""
cmd = [self.__command, src, "-o", dst] + self.__linker_flags + self.get_library_directory_list() + self.get_library_list() + extra_args + self.__linker_script + self.__linker_flags_extra
(so, se) = run_command(cmd)
if 0 < len(se) and is_verbose():
print(se)
return so
def get_library_name(self, op):
"""Get actual name of library."""
if op.startswith("/"):
return op
# Check if the library is specified verbatim. If yes, no need to expand.
if re.match(r'lib.+\.so(\..*)?', op):
return op
libname = "lib%s.so" % (op)
# Search in library directories only.
for ii in self.__library_directories:
current_libname = locate(ii, libname)
if not current_libname:
continue
# Check if the supposed shared library is a linker script.
if file_is_ascii_text(current_libname):
ret = read_linker_script_library_name(current_libname)
if ret:
if is_verbose():
print(
"'%s' is a linker script, actual library name: '%s'"
% (libname, ret))
return ret
# Stop at first match.
break
return libname
def generate_source(self, required_symbols):
"""Generate C source that contains definitions for given symbols."""
headers = set()
prototypes = []
source = []
compiled_symbol_names = []
ii = 0
while ii < len(required_symbols):
name = required_symbols[ii]
if name in self.__symbols:
sym = self.__symbols[name]
# Add all dependencies to required symbols list.
for jj in sym.get_dependencies():
if not jj in required_symbols:
required_symbols += [jj]
# Extend collected source data.
headers = headers.union(sym.get_headers())
prototypes += sym.get_prototypes()
source += [sym.get_source()]
compiled_symbol_names += [name]
ii += 1
if not source:
return None
if is_verbose():
print("%i extra symbols required: %s" %
(len(compiled_symbol_names), str(compiled_symbol_names)))
subst = {
"HEADERS": "\n".join(map(lambda x: "#include <%s>" % (x),
headers)),
"PROTOTYPES": "\n\n".join(prototypes),
"SOURCE": "\n\n".join(source)
}
return g_template_extra_source.format(subst)
def generate_fake_bss(self, assembler, und_symbols = None, elfling = None):
"""Remove local labels that would seem to generate .bss, make a fake .bss section."""
bss = AssemblerSectionBss()
for ii in self.__sections:
while True:
entry = ii.extract_bss(und_symbols)
if not entry:
break
if not entry.is_und_symbol():
bss.add_element(entry)
if elfling:
bss.add_element(AssemblerBssElement(ELFLING_WORK, elfling.get_work_size()))
bss_size = bss.get_size()
if 0 < bss.get_alignment():
pt_load_string = ", second PT_LOAD required"
else:
pt_load_string = ", one PT_LOAD sufficient"
if is_verbose():
outstr = "Constructed fake .bss segement: "
if 1073741824 < bss_size:
print("%s%1.1f Gbytes%s" % (outstr, float(bss_size) / 1073741824.0, pt_load_string))
elif 1048576 < bss_size:
print("%s%1.1f Mbytes%s" % (outstr, float(bss_size) / 1048576.0, pt_load_string))
elif 1024 < bss_size:
print("%s%1.1f kbytes%s" % (outstr, float(bss_size) / 1024.0, pt_load_string))
else:
print("%s%u bytes%s" % (outstr, bss_size, pt_load_string))
self.add_sections(bss)
return bss
def get_library_name(self, op):
"""Get actual name of library."""
if op.startswith("/"):
return op
# Check if the library is specified verbatim. If yes, no need to expand.
if re.match(r'lib.+\.so(\..*)?', op):
return op
libname = "lib%s.so" % (op)
# Search in library directories only.
for ii in self.__library_directories:
current_libname = locate(ii, libname)
if not current_libname:
continue
# Check if the supposed shared library is a linker script.
if file_is_ascii_text(current_libname):
fd = open(current_libname, "r")
match = re.search(r'GROUP\s*\(\s*(\S+)\s+', fd.read(),
re.MULTILINE)
fd.close()
if match:
ret = os.path.basename(match.group(1))
if is_verbose():
print("Using shared library '%s' instead of '%s'." %
(ret, libname))
return ret
# Stop at first match.
break
return libname
def crunch_align(self):
"""Replace all .align declarations with minimal byte alignment."""
desired = int(PlatformVar("align"))
adjustments = []
for ii in range(len(self.__content)):
line = self.__content[ii]
match = re.match(r'(\s*)\.align\s+(\d+).*', line, re.I)
if not match:
continue
# Compiler thinking aligning to less than desired platform alignment is probably ok.
align = get_align_bytes(int(match.group(2)))
if align <= desired:
continue
# Some alignment directives are necessary due to data access.
if not can_minimize_align(align):
continue
self.__content[ii] = "%s.balign %i\n" % (match.group(1), desired)
adjustments += ["%i -> %i" % (align, desired)]
# Data sections may be reshuffled and require minimal align as first line.
if self.__name in ["data", "rodata"]:
match = re.match(r'(\s*)\.b?align\s.*', self.__content[0], re.I)
if not match:
first_align = "\t.balign %i\n" % (int(PlatformVar("align")))
self.__content.insert(0, first_align)
if is_verbose() and adjustments:
print("Alignment adjustment(%s): %s" %
(self.get_name(), ", ".join(adjustments)))
def generate_source(self, required_symbols):
"""Generate C source that contains definitions for given symbols."""
headers = set()
prototypes = []
source = []
compiled_symbol_names = []
ii = 0
while ii < len(required_symbols):
name = required_symbols[ii]
if name in self.__symbols:
sym = self.__symbols[name]
# Add all dependencies to required symbols list.
for jj in sym.get_dependencies():
if jj not in required_symbols:
required_symbols += [jj]
# Extend collected source data.
headers = headers.union(sym.get_headers())
prototypes += sym.get_prototypes()
source += [sym.get_source()]
compiled_symbol_names += [name]
ii += 1
if not source:
return None
if is_verbose():
print("%i extra symbols required: %s" % (len(compiled_symbol_names), str(compiled_symbol_names)))
subst = {"HEADERS": "\n".join(map(lambda x: "#include <%s>" % (x), headers)),
"PROTOTYPES": "\n\n".join(prototypes),
"SOURCE": "\n\n".join(source)
}
return g_template_extra_source.format(subst)
def add_source(self, fname):
"""Add source from an assembler file."""
fd = open(fname, "r")
lines = fd.readlines()
fd.close()
current_section = AssemblerSection("text")
sectionre = re.compile(r'^\s*\.section\s+\"?\.([a-zA-Z0-9_]+)[\.\s]')
directivere = re.compile(r'^\s*\.(bss|data|rodata|text)')
for ii in lines:
# Try both expressions first.
match = sectionre.match(ii)
if not match:
match = directivere.match(ii)
# If match, start new section.
if match:
self.add_sections(current_section)
current_section = AssemblerSection(match.group(1), ii)
else:
current_section.add_content(ii)
if not current_section.empty():
self.add_sections(current_section)
if is_verbose():
section_names = map(lambda x: x.get_name(), self.__sections)
print("%i sections in '%s': %s" %
(len(self.__sections), fname, str(section_names)))
def parse(self):
"""Parse all source files."""
# First, assemble glsl chains.
source_dict = {}
for ii in self.__sources:
if ii.isCommonChainName():
continue
chain_name = ii.getChainName()
if chain_name in source_dict:
source_dict[chain_name].addSource(ii)
else:
source_dict[chain_name] = GlslSourceChain(ii)
for ii in self.__sources:
if not ii.isCommonChainName():
continue
for jj in source_dict.keys():
source_chain = source_dict[jj]
if source_chain.isSourceSlotFree(ii):
source_chain.addSource(ii)
self.__chains = source_dict.values()
if is_verbose():
print("GLSL source chains: %s" % (" ; ".join(map(lambda x: str(x), self.__chains))))
# Run parse process on sources.
for ii in self.__sources:
ii.parse()
def get_library_name(self, linker):
"""Get linkable library object name."""
libname = linker.get_library_name(self.__library.get_name())
if libname != self.__library.get_name() and is_verbose():
print("Using shared library '%s' instead of '%s'." %
(str(libname), self.__library.get_name()))
return libname
def generate_fake_bss(self, assembler, und_symbols=None, elfling=None):
"""Remove local labels that would seem to generate .bss, make a fake .bss section."""
bss = AssemblerSectionBss()
for ii in self.__sections:
while True:
entry = ii.extract_bss(und_symbols)
if not entry:
break
if not entry.is_und_symbol():
bss.add_element(entry)
if elfling:
bss.add_element(AssemblerBssElement(ELFLING_WORK, elfling.get_work_size()))
bss_size = bss.get_size()
if 0 < bss.get_alignment():
pt_load_string = ", second PT_LOAD required"
else:
pt_load_string = ", one PT_LOAD sufficient"
if is_verbose():
outstr = "Constructed fake .bss segement: "
if 1073741824 < bss_size:
print("%s%1.1f Gbytes%s" % (outstr, float(bss_size) / 1073741824.0, pt_load_string))
elif 1048576 < bss_size:
print("%s%1.1f Mbytes%s" % (outstr, float(bss_size) / 1048576.0, pt_load_string))
elif 1024 < bss_size:
print("%s%1.1f kbytes%s" % (outstr, float(bss_size) / 1024.0, pt_load_string))
else:
print("%s%u bytes%s" % (outstr, bss_size, pt_load_string))
self.add_sections(bss)
return bss
def merge(self, op):
"""Attempt to merge with given segment."""
highest_mergable = 0
(head_src, bytestream_src) = self.deconstruct_tail()
(bytestream_dst, tail_dst) = op.deconstruct_head()
for ii in range(min(len(bytestream_src), len(bytestream_dst))):
mergable = True
for jj in range(ii + 1):
if not bytestream_src[-ii - 1 + jj].mergable(
bytestream_dst[jj]):
mergable = False
break
if mergable:
highest_mergable = ii + 1
if 0 >= highest_mergable:
return False
if is_verbose():
print("Merging headers %s and %s at %i bytes." %
(self.__name, op.__name, highest_mergable))
for ii in range(highest_mergable):
bytestream_src[-highest_mergable + ii].merge(bytestream_dst[ii])
bytestream_dst[0:highest_mergable] = []
self.reconstruct(head_src + bytestream_src)
op.reconstruct(bytestream_dst + tail_dst)
return True
def replace_osname(repl_osname, reason):
"""Replace osname with given string."""
global g_osname
if g_osname == repl_osname:
return
if is_verbose():
print("%stargeting osname '%s' instead of '%s'" % (reason, repl_osname, g_osname))
g_osname = repl_osname
def replace_osarch(repl_osarch, reason):
"""Replace osarch with given string."""
global g_osarch
if g_osarch == repl_osarch:
return
if is_verbose():
print("%stargeting osarch '%s' instead of '%s'" % (reason, repl_osarch, g_osarch))
g_osarch = repl_osarch
def compile_asm(self, src, dst, whole_program=False):
"""Compile a file into assembler source."""
cmd = [self.get_command(), "-S", src, "-o", dst] + self.__standard + self.__compiler_flags + self._compiler_flags_extra + self._definitions + self._include_directories
if whole_program:
cmd += self.__compiler_flags_generate_asm
(so, se) = run_command(cmd)
if 0 < len(se) and is_verbose():
print(se)
def compile_asm(self, src, dst, whole_program=False):
"""Compile a file into assembler source."""
cmd = [self.get_command(), "-S", src, "-o", dst] + self.__standard + self.__compiler_flags + self._compiler_flags_extra + self._definitions + self._include_directories
if whole_program:
cmd += self.__compiler_flags_generate_asm
(so, se) = run_command(cmd)
if 0 < len(se) and is_verbose():
print(se)
def replace_osname(repl_osname, reason):
"""Replace osname with given string."""
global g_osname
if g_osname == repl_osname:
return
if is_verbose():
print("%stargeting osname '%s' instead of '%s'" %
(reason, repl_osname, g_osname))
g_osname = repl_osname
def replace_osarch(repl_osarch, reason):
"""Replace osarch with given string."""
global g_osarch
if g_osarch == repl_osarch:
return
if is_verbose():
print("%stargeting osarch '%s' instead of '%s'" %
(reason, repl_osarch, g_osarch))
g_osarch = repl_osarch
def detectType(self):
"""Try to detect chain name and type from filename."""
(self.__chain, self.__type) = detect_shader_type(self.__basename)
if is_verbose():
output_message = "Shader file '%s' type" % (self.__filename)
if self.__type:
print(output_message + (": '%s'" % (self.__type)))
else:
print(output_message + " not detected, assuming generic.")
def compile_and_link(self, src, dst):
"""Compile and link a file directly."""
cmd = [
self.get_command(), src, "-o", dst
] + self.__standard + self.__compiler_flags + self._compiler_flags_extra + self._definitions + self._include_directories + self.get_linker_flags(
) + self.get_library_directory_list() + self.get_library_list()
(so, se) = run_command(cmd)
if 0 < len(se) and is_verbose():
print(se)
def format(self, force):
"""Return formatted output."""
if not self.__rename:
if force:
if is_verbose():
print("WARNING: %s not locked" % (self))
return self.__name
return ""
return self.__rename
def preprocess(self, op):
"""Preprocess a file, return output."""
args = [self.get_command(), op] + self._compiler_flags_extra + self._definitions + self._include_directories
if self.is_msvc():
args += ["/E"]
(so, se) = run_command(args)
if 0 < len(se) and is_verbose():
print(se)
return so
def crunch(self):
"""Remove all offending content."""
self.crunch_align()
self.crunch_redundant()
if osarch_is_amd64() or osarch_is_ia32():
self.crunch_amd64_ia32()
elif is_verbose():
print("WARNING: no platform-dependent crunch for architecture '%s'" % g_osarch)
self.__tag = None
def write(self):
"""Write compressed output."""
fd = open(self.__output_name, "w")
if not fd:
raise RuntimeError("could not write GLSL header '%s'" % (self.__output_name))
fd.write(self.generateHeaderOutput())
fd.close()
if is_verbose():
print("Wrote GLSL header: '%s' => '%s'" % (self.__variable_name, self.__output_name))
def format(self, force):
"""Return formatted output."""
if not self.__rename:
if force:
if is_verbose():
print("WARNING: %s not locked" % (self))
return self.__name
return ""
return self.__rename
def preprocess(self, op):
"""Preprocess a file, return output."""
args = [self.get_command(), op] + self._compiler_flags_extra + self._definitions + self._include_directories
if self.command_basename_startswith("cl."):
args += ["/E"]
(so, se) = run_command(args)
if 0 < len(se) and is_verbose():
print(se)
return so
def link_binary(self, src, dst):
"""Link a binary file with no bells and whistles."""
cmd = [self.__command,
"--entry=" + str(PlatformVar("entry"))] + listify(src) + [
"-o", dst
] + self.__linker_script + self.__linker_flags_extra
(so, se) = run_command(cmd)
if 0 < len(se) and is_verbose():
print(se)
return so
def crunch_entry_push(self, op):
"""Crunch amd64/ia32 push directives from given line listing."""
lst = self.want_label(op)
if not lst:
return
ii = lst[0] + 1
jj = ii
stack_decrement = 0
reinstated_lines = []
while True:
current_line = self.__content[jj]
match = re.match(r'\s*(push\w).*%(\w+)', current_line, re.IGNORECASE)
if match:
stack_decrement += get_push_size(match.group(1))
jj += 1
continue
# Preserve comment lines as they are.
match = re.match(r'^\s*[#;].*', current_line, re.IGNORECASE)
if match:
reinstated_lines += [current_line]
jj += 1
continue
# Some types of lines can be in the middle of pushing.
if is_reinstate_line(current_line):
reinstated_lines += [current_line]
jj += 1
continue
# Stop at stack decrement.
match = re.match(r'\s*sub.*\s+[^\d]*(\d+),\s*%(rsp|esp)', current_line, re.IGNORECASE)
if match:
total_decrement = int(match.group(1)) + stack_decrement
# As per gcc ABI, align to 16 bytes on 64-bit architectures.
if osarch_is_64_bit() and ((total_decrement & 0xF) != 0):
total_decrement += 0x10 - (total_decrement & 0xF)
self.__content[jj] = re.sub(r'\d+', str(total_decrement), current_line)
break
# Do nothing if suspicious instruction is found.
if is_verbose():
print("Unknown header instruction found, aborting erase: '%s'" % (current_line.strip()))
break
if is_verbose():
print("Erasing function header from '%s': %i lines" % (op, jj - ii - len(reinstated_lines)))
self.__content[ii:jj] = reinstated_lines
def link(self, src, dst, extra_args=[]):
"""Link a file."""
cmd = [
self.__command, src, "-o", dst
] + self.__linker_flags + self.get_library_directory_list(
) + self.get_library_list(
) + extra_args + self.__linker_script + self.__linker_flags_extra
(so, se) = run_command(cmd)
if 0 < len(se) and is_verbose():
print(se)
return so
def get_extra_library_directories(self):
"""Determine extra library requirements for the compiler."""
ret = []
if self.is_gcc():
(so, se) = run_command([self.get_command(), "-v"])
match = re.search(r'COLLECT_LTO_WRAPPER\s*\=\s*([^\n]+)', se, re.I | re.M)
if match:
ret += [os.path.dirname(match.group(1))]
if is_verbose() and ret:
print("Compiler '%s' requires additional library directories: %s" % (self.get_command_basename(), str(ret)))
return ret
def write(self):
"""Write compressed output."""
fd = open(self.__output_name, "w")
if not fd:
raise RuntimeError("could not write GLSL header '%s'" %
(self.__output_name))
fd.write(self.generateHeaderOutput())
fd.close()
if is_verbose():
print("Wrote GLSL header: '%s' => '%s'" %
(self.__variable_name, self.__output_name))
def extract_shader_payload(preprocessor, src, dst):
"""Extract only the quoted content and write a file."""
text = preprocessor.preprocess(src)
match = re.match(r'.*char[^"]+"(.*)"\s*;[^"]+', text, re.MULTILINE | re.DOTALL)
if not match:
raise RuntimeError("could not extract shader blob")
text = re.sub(r'"\s*\n\s*"', "", match.group(1))
fd = open(dst, "w")
fd.write(text.replace("\\n", "\n"))
fd.close()
if is_verbose():
print("Wrote shader payload: '%s'" % (dst))
def format(self, force):
"""Return formatted output."""
ret = "."
if self.__swizzle:
if not self.__swizzle_export:
if force:
if is_verbose():
print("WARNING: %s swizzle status unconfirmed" % (str(self)))
return ret + self.__name.format(force)
return ""
return ret + self.generateSwizzle()
return ret + self.__name.format(force)
def extract_shader_payload(preprocessor, src, dst):
"""Extract only the quoted content and write a file."""
text = preprocessor.preprocess(src)
match = re.match(r'.*char[^"]+"(.*)"\s*;[^"]+', text, re.MULTILINE | re.DOTALL)
if not match:
raise RuntimeError("could not extract shader blob")
text = re.sub(r'"\s*\n\s*"', "", match.group(1))
fd = open(dst, "w")
fd.write(text.replace("\\n", "\n"))
fd.close()
if is_verbose():
print("Wrote shader payload: '%s'" % (dst))
def generate_linker_script(self, dst, modify_start=False):
"""Get linker script from linker, improve it, write improved linker script to given file."""
(so, se) = run_command([self.__command, "--verbose"] + self.__linker_flags_extra)
if 0 < len(se) and is_verbose():
print(se)
# Linker script is the block of code between lines of multiple '=':s.
match = re.match(r'.*\n=+\s*\n(.*)\n=+\s*\n.*', so, re.DOTALL)
if not match:
raise RuntimeError("could not extract script from linker output")
ld_script = match.group(1)
# Remove unwanted symbol definitions one at a time.
unwanted_symbols = ["__bss_end__", "__bss_start__", "__end__", "__bss_start", "_bss_end__", "_edata", "_end"]
for ii in unwanted_symbols:
ld_script = re.sub(r'\n([ \f\r\t\v]+)(%s)(\s*=[^\n]+)\n' % (ii), r'\n\1/*\2\3*/\n', ld_script, re.MULTILINE)
ld_script = re.sub(r'SEGMENT_START\s*\(\s*(\S+)\s*,\s*\d*x?\d+\s*\)', r'SEGMENT_START(\1, %s)' % (str(PlatformVar("entry"))), ld_script, re.MULTILINE)
if modify_start:
ld_script = re.sub(r'(SEGMENT_START.*\S)\s*\+\s*SIZEOF_HEADERS\s*;', r'\1;', ld_script, re.MULTILINE)
fd = open(dst, "w")
fd.write(ld_script)
fd.close()
if is_verbose():
print("Wrote linker script '%s'." % (dst))
return ld_script
def format(self, substitutions = None):
"""Return formatted output."""
ret = self.__content
if substitutions:
for kk in substitutions:
vv = substitutions[kk].replace("\\", "\\\\")
(ret, num) = re.subn(r'\[\[\s*%s\s*\]\]' % (kk), vv, ret)
if not num:
print("WARNING: substitution '%s' has no matches" % (kk))
unmatched = list(set(re.findall(r'\[\[([^\]]+)\]\]', ret)))
(ret, num) = re.subn(r'\[\[[^\]]+\]\]', "", ret)
if num and is_verbose():
print("Template substitutions not matched: %s (%i)" % (str(unmatched), num))
return ret
def crunch_jump_pop(self, op):
"""Crunch popping before a jump."""
lst = self.want_line(r'\s*(jmp\s+%s)\s+.*' % (op))
if not lst:
return
ii = lst[0]
jj = ii - 1
while True:
if (0 > jj) or not re.match(r'\s*(pop\S).*', self.__content[jj], re.IGNORECASE):
if is_verbose():
print("Erasing function footer before jump to '%s': %i lines" % (op, ii - jj - 1))
self.erase(jj + 1, ii)
break
jj -= 1
def format(self, substitutions=None):
"""Return formatted output."""
ret = self.__content
if substitutions:
for kk in substitutions:
vv = substitutions[kk].replace("\\", "\\\\")
(ret, num) = re.subn(r'\[\[\s*%s\s*\]\]' % (kk), vv, ret)
if not num:
print("WARNING: substitution '%s' has no matches" % (kk))
unmatched = list(set(re.findall(r'\[\[([^\]]+)\]\]', ret)))
(ret, num) = re.subn(r'\[\[[^\]]+\]\]', "", ret)
if num and is_verbose():
print("Template substitutions not matched: %s (%i)" % (str(unmatched), num))
return ret
def crunch_jump_pop(self, op):
"""Crunch popping before a jump."""
lst = self.want_line(r'\s*(jmp\s+%s)\s+.*' % (op))
if not lst:
return
ii = lst[0]
jj = ii - 1
while True:
if (0 > jj) or not re.match(r'\s*(pop\S).*', self.__content[jj], re.IGNORECASE):
if is_verbose():
print("Erasing function footer before jump to '%s': %i lines" % (op, ii - jj - 1))
self.erase(jj + 1, ii)
break
jj -= 1
def __init__(self, lst, explicit):
"""Constructor."""
GlslBlock.__init__(self)
self.__explicit = explicit
self.__squashable = False
self.__allow_squash = False
# Check for degenerate scope.
if (1 == len(lst)) and is_glsl_block_declaration(lst[0]):
raise RuntimeError("scope with only block '%s' is degenerate" % (lst[0].format(True)))
# Check for empty scope (likely an error).
if 0 >= len(lst):
if is_verbose():
print("WARNING: empty scope")
# Hierarchy.
self.addChildren(lst)
def crunch_entry_push(self, op):
"""Crunch amd64/ia32 push directives from given line listing."""
lst = self.want_label(op)
if not lst:
return
ii = lst[0] + 1
jj = ii
stack_decrement = 0
stack_save_decrement = 0
reinstated_lines = []
while True:
current_line = self.__content[jj]
match = re.match(r'\s*(push\w).*%(\w+)', current_line, re.IGNORECASE)
if match:
if is_stack_save_register(match.group(2)):
stack_save_decrement += get_push_size(match.group(1))
else:
stack_decrement += get_push_size(match.group(1))
jj += 1
continue;
# Preserve comment lines as they are.
match = re.match(r'^\s*[#;].*', current_line, re.IGNORECASE)
if match:
reinstated_lines += [current_line]
jj += 1
continue
# Saving stack pointer or sometimes initializing edx seem to be within pushing.
match = re.match(r'\s*mov\w\s+%\w+,\s*%(rbp|ebp|edx).*', current_line, re.IGNORECASE)
if match:
if is_stack_save_register(match.group(1)):
stack_save_decrement = 0
reinstated_lines += [current_line]
jj += 1
continue;
# Some types of lines can be in the middle of pushing.
if is_reinstate_line(current_line):
reinstated_lines += [current_line]
jj += 1
continue
match = re.match(r'\s*sub.*\s+[^\d]*(\d+),\s*%(rsp|esp)', current_line, re.IGNORECASE)
if match:
total_decrement = int(match.group(1)) + stack_decrement + stack_save_decrement
self.__content[jj] = re.sub(r'\d+', str(total_decrement), current_line)
break
if is_verbose():
print("Erasing function header from '%s': %i lines" % (op, jj - ii - len(reinstated_lines)))
self.erase(ii, jj)
self.__content[ii:ii] = reinstated_lines
def write(self, op, assembler, section_names = None):
"""Write an output assembler file or append to an existing file."""
output = self.generate_file_output(section_names)
if not output:
return False
if isinstance(op, str):
fd = open(op, "w")
fd.write(output)
fd.close()
if is_verbose():
print("Wrote assembler source: '%s'" % (op))
else:
prefix = assembler.format_block_comment("sections '%s'" % (str(section_names)))
op.write(prefix)
op.write(output)
return True
def crunch_amd64(self):
"""Perform platform-dependent crunching."""
self.crunch_entry_push("_start")
self.crunch_entry_push(ELFLING_UNCOMPRESSED)
self.crunch_jump_pop(ELFLING_UNCOMPRESSED)
lst = self.want_line(r'\s*(int\s+\$0x3|syscall)\s+.*')
if lst:
ii = lst[0] + 1
jj = ii
while True:
if (jj >= len(self.__content)) or (not can_erase_footer(self.__content[jj])):
if is_verbose():
print("Erasing function footer after '%s': %i lines" % (lst[1], jj - ii))
self.erase(ii, jj)
break
jj += 1
def crunch_ia32(self):
"""Perform platform-dependent crunching."""
self.crunch_entry_push("_start")
self.crunch_entry_push(ELFLING_UNCOMPRESSED)
self.crunch_jump_pop(ELFLING_UNCOMPRESSED)
lst = self.want_line(r'\s*int\s+\$(0x3|0x80)\s+.*')
if lst:
ii = lst[0] + 1
jj = ii
while True:
if len(self.__content) <= jj or re.match(r'\s*\S+\:\s*', self.__content[jj]):
if is_verbose():
print("Erasing function footer after interrupt '%s': %i lines" % (lst[1], jj - ii))
self.erase(ii, jj)
break
jj += 1
def detectType(self):
"""Try to detect type from filename."""
self.__type = None
stub = r'.*[._\-\s]%s[._\-\s].*'
if re.match(stub % ("frag"), self.__filename, re.I) or re.match(stub % ("fragment"), self.__filename, re.I):
self.__type = "fragment"
elif re.match(stub % ("geom"), self.__filename, re.I) or re.match(stub % ("geometry"), self.__filename, re.I):
self.__type = "geometry"
elif re.match(stub % ("vert"), self.__filename, re.I) or re.match(stub % ("vertex"), self.__filename, re.I):
self.__type = "vertex"
if is_verbose():
output_message = "Shader file '%s' type" % (self.__filename)
if self.__type:
print(output_message + (": '%s'" % (self.__type)))
else:
print(output_message + " not detected, assuming generic.")
def write(self, op, assembler, section_names=None):
"""Write an output assembler file or append to an existing file."""
output = self.generate_file_output(section_names)
if not output:
return False
if isinstance(op, str):
fd = open(op, "w")
fd.write(output)
fd.close()
if is_verbose():
print("Wrote assembler source: '%s'" % (op))
else:
prefix = assembler.format_block_comment("sections '%s'" % (str(section_names)))
op.write(prefix)
op.write(output)
return True
def crunch_align(self):
"""Replace all .align declarations with minimal byte alignment."""
desired = int(PlatformVar("align"))
adjustments = []
for ii in range(len(self.__content)):
line = self.__content[ii]
match = re.match(r'(\s*)\.align\s+(\d+).*', line)
if not match:
continue
# Get actual align byte count.
align = get_align_bytes(int(match.group(2)))
if align == desired:
continue
# Some alignment directives are necessary due to data access.
if not can_minimize_align(align):
continue
self.__content[ii] = "%s.balign %i\n" % (match.group(1), desired)
adjustments += ["%i -> %i" % (align, desired)]
if is_verbose() and adjustments:
print("Alignment adjustment(%s): %s" % (self.get_name(), ", ".join(adjustments)))
def merge(self, op):
"""Attempt to merge with given segment."""
highest_mergable = 0
(head_src, bytestream_src) = self.deconstruct_tail()
(bytestream_dst, tail_dst) = op.deconstruct_head()
for ii in range(min(len(bytestream_src), len(bytestream_dst))):
mergable = True
for jj in range(ii + 1):
if not bytestream_src[-ii - 1 + jj].mergable(bytestream_dst[jj]):
mergable = False
break
if mergable:
highest_mergable = ii + 1
if 0 >= highest_mergable:
return False
if is_verbose():
print("Merging headers %s and %s at %i bytes." % (self.__name, op.__name, highest_mergable))
for ii in range(highest_mergable):
bytestream_src[-highest_mergable + ii].merge(bytestream_dst[ii])
bytestream_dst[0:highest_mergable] = []
self.reconstruct(head_src + bytestream_src)
op.reconstruct(bytestream_dst + tail_dst)
return True
def add_source(self, fname):
"""Add source from an assembler file."""
fd = open(fname, "r")
lines = fd.readlines()
fd.close()
current_section = AssemblerSection("text")
sectionre = re.compile(r'^\s*\.section\s+\"?\.([a-zA-Z0-9_]+)[\.\s]')
directivere = re.compile(r'^\s*\.(bss|data|rodata|text)')
for ii in lines:
# Try both expressions first.
match = sectionre.match(ii)
if not match:
match = directivere.match(ii)
# If match, start new section.
if match:
self.add_sections(current_section)
current_section = AssemblerSection(match.group(1), ii)
else:
current_section.add_content(ii)
if not current_section.empty():
self.add_sections(current_section)
if is_verbose():
section_names = map(lambda x: x.get_name(), self.__sections)
print("%i sections in '%s': %s" % (len(self.__sections), fname, str(section_names)))
def sort_sections(self, assembler, data_in_front=True):
"""Sort sections into an order that is more easily compressible."""
text_sections = []
rodata_sections = []
data_sections = []
other_sections = []
for ii in self.__sections:
if "text" == ii.get_name():
text_sections += [ii]
elif "rodata" == ii.get_name():
rodata_sections += [ii]
elif "data" == ii.get_name():
data_sections += [ii]
else:
other_sections += [ii]
text_section_str = []
rodata_section_str = []
data_section_str = []
other_section_str = []
if text_sections:
text_section_str += ["%i text" % (len(text_sections))]
if rodata_sections:
rodata_section_str = ["%i rodata" % (len(rodata_sections))]
if data_sections:
data_section_str += ["%i data" % (len(data_sections))]
if other_sections:
other_section_str = [", ".join(map(lambda x: x.get_name(), other_sections))]
# Sort data either in front or in the back.
if data_in_front:
section_str = rodata_section_str + data_section_str + text_section_str + other_section_str
self.__sections = rodata_sections + data_sections + text_sections + other_sections
else:
section_str = text_section_str + rodata_section_str + data_section_str + other_section_str
self.__sections = text_sections + rodata_sections + data_sections + other_sections
if is_verbose():
print("Sorted sections: " + ", ".join(filter(lambda x: x, section_str)))
def main():
"""Main function."""
default_preprocessor_list = ["cpp", "clang-cpp"]
preprocessor = None
parser = argparse.ArgumentParser(usage = "GLSL minifying test.", formatter_class = CustomHelpFormatter, add_help = False)
parser.add_argument("-h", "--help", action = "store_true", help = "Print this help string and exit.")
parser.add_argument("--preprocessor", default = None, help = "Try to use given preprocessor executable as opposed to autodetect.")
parser.add_argument("-v", "--verbose", action = "store_true", help = "Print more info about what is being done.")
parser.add_argument("source", default = [], nargs = "*", help = "Source file(s) to process.")
args = parser.parse_args()
preprocessor = args.preprocessor
if args.help:
print(parser.format_help().strip())
return 0
# Verbosity.
if args.verbose:
set_verbose(True)
# Source files to process.
if not args.source:
raise RuntimeError("no source files to process")
source_files = []
for ii in args.source:
if re.match(r'.*\.(glsl|vert|geom|frag)$', ii, re.I):
source_files += [ii]
else:
raise RuntimeError("unknown source file: '%s'" % (ii))
dl = find_executable("dnload.py", "dnload")
if is_verbose():
print("found dnload: '%s'" % (dl))
sm = find_executable("shader_minifier.exe", "Shader_Minifier")
if is_verbose():
print("found shader_minifier: '%s'" % (sm))
# Find preprocessor.
if preprocessor:
if not executable_check(preprocessor):
raise RuntimeError("could not use supplied preprocessor '%s'" % (preprocessor))
else:
preprocessor_list = default_preprocessor_list
if os.name == "nt":
preprocessor_list = ["cl.exe"] + preprocessor_list
preprocessor = executable_search(preprocessor_list, "preprocessor")
if not preprocessor:
raise RuntimeError("suitable preprocessor not found")
preprocessor = Preprocessor(preprocessor)
for ii in source_files:
fname = "/tmp/" + os.path.basename(ii)
fname_dn = fname + ".dnload"
fname_dn_in = fname_dn + ".h"
fname_dn_out = fname_dn + ".payload"
fname_sm = fname + ".shaderminifier"
fname_sm_in = fname_sm + ".h"
fname_sm_out = fname_sm + ".payload"
run_command(["python", dl, ii, "-o", fname_dn_in])
if is_verbose():
print("Wrote dnload -minified shader: '%s'" % (fname_dn_in))
run_command(["mono", sm, ii, "-o", fname_sm_in])
if is_verbose():
print("Wrote shader_minifier -minified shader: '%s'" % (fname_sm_in))
extract_shader_payload(preprocessor, fname_dn_in, fname_dn_out)
extract_shader_payload(preprocessor, fname_sm_in, fname_sm_out)
compress_file("lzma", fname_dn_out, fname_dn + ".lzma")
#compress_file("xz", fname_dn_out, fname_dn + ".xz")
compress_file("lzma", fname_sm_out, fname_sm + ".lzma")
#compress_file("xz", fname_sm_out, fname_sm + ".xz")
return 0
def compile_and_link(self, src, dst):
"""Compile and link a file directly."""
cmd = [self.get_command(), src, "-o", dst] + self.__standard + self.__compiler_flags + self._compiler_flags_extra + self._definitions + self._include_directories + self.get_linker_flags() + self.get_library_directory_list() + self.get_library_list()
(so, se) = run_command(cmd)
if 0 < len(se) and is_verbose():
print(se)