def print_named_filedata(self, filename):
name = os.path.split(filename)[1]
size = os.path.getsize(filename)
self.start_sprite(0, 0xfd)
self.sprite_output.start_sprite(8 + 3 + len(name) + 1 + size)
self.sprite_output.print_dword(self.sprite_num)
self.sprite_output.print_dword(3 + len(name) + 1 + size)
self.sprite_output.print_byte(0xff)
self.sprite_output.print_byte(0xff)
self.sprite_output.print_byte(len(name))
self.print_string(
name, force_ascii=True, final_zero=True,
stream=self.sprite_output) # ASCII filenames seems sufficient.
fp = open(generic.find_file(filename), 'rb')
while True:
data = fp.read(1024)
if len(data) == 0: break
for d in data:
self.sprite_output.print_byte(d)
fp.close()
self.sprite_output.end_sprite()
self.end_sprite()
def read_extra_commands(custom_tags_file):
"""
@param custom_tags_file: Filename of the custom tags file.
@type custom_tags_file: C{str}
"""
if not os.access(custom_tags_file, os.R_OK):
# Failed to open custom_tags.txt, ignore this
return
line_no = 0
with open(generic.find_file(custom_tags_file), "r", encoding="utf-8") as fh:
for line in fh:
line_no += 1
line = line.strip()
if len(line) == 0 or line[0] == "#":
continue
i = line.find(":")
if i == -1:
raise generic.ScriptError("Line has no ':' delimiter.", generic.LinePosition(custom_tags_file, line_no))
name = line[:i].strip()
value = line[i + 1 :]
if name in commands:
generic.print_warning(
'Overwriting existing tag "' + name + '".', generic.LinePosition(custom_tags_file, line_no)
)
commands[name] = {"unicode": value}
if is_ascii_string(value):
commands[name]["ascii"] = value
def validate_size(self):
"""
Check if xpos/ypos/xsize/ysize are already set and if not, set them
to 0,0,image_width,image_height.
"""
if self.xpos is None:
with Image.open(generic.find_file(self.file.value)) as im:
self.xpos = expression.ConstantNumeric(0)
self.ypos = expression.ConstantNumeric(0)
self.xsize = expression.ConstantNumeric(im.size[0])
self.ysize = expression.ConstantNumeric(im.size[1])
self.check_sprite_size()
if self.mask_pos is None:
self.mask_pos = (self.xpos, self.ypos)
def open_image_file(self, filename):
"""
Obtain a handle to an image file
@param filename: Name of the file
@type filename: C{str}
@return: Image file
@rtype: L{Image}
"""
if filename in self.cached_image_files:
im = self.cached_image_files[filename]
else:
im = Image.open(generic.find_file(filename))
self.cached_image_files[filename] = im
return im
def parse_file(filename, default):
"""
Read and parse a single language file.
@param filename: The filename of the file to parse.
@type filename: C{str}
@param default: True iff this is the default language.
@type default: C{bool}
"""
lang = Language(False)
try:
with open(generic.find_file(filename), "r", encoding="utf-8") as fh:
for idx, line in enumerate(fh):
pos = generic.LinePosition(filename, idx + 1)
line = line.rstrip("\n\r").lstrip("\uFEFF")
# The default language is processed twice here. Once as fallback langauge
# and once as normal language.
if default:
default_lang.handle_string(line, pos)
lang.handle_string(line, pos)
except UnicodeDecodeError:
pos = generic.LanguageFilePosition(filename)
if default:
raise generic.ScriptError("The default language file contains non-utf8 characters.", pos)
generic.print_warning("Language file contains non-utf8 characters. Ignoring (part of) the contents.", pos)
except generic.ScriptError as err:
if default:
raise
generic.print_warning(err.value, err.pos)
else:
if lang.langid is None:
generic.print_warning(
"Language file does not contain a ##grflangid pragma", generic.LanguageFilePosition(filename)
)
else:
for lng in langs:
if lng[0] == lang.langid:
msg = "Language file has the same ##grflangid (with number {:d}) as another language file".format(
lang.langid
)
raise generic.ScriptError(msg, generic.LanguageFilePosition(filename))
langs.append((lang.langid, lang))
def nml(inputfile, input_filename, output_debug, outputfiles, start_sprite_num,
compress_grf, crop_sprites, enable_cache, forced_palette, md5_filename,
rebuild_parser, debug_parser):
"""
Compile an NML file.
@param inputfile: File handle associated with the input file.
@type inputfile: C{File}
@param input_filename: Filename of the input file, C{None} if receiving from L{sys.stdin}
@type input_filename: C{str} or C{None}
@param outputfiles: Output streams to write to.
@type outputfiles: C{List} of L{output_base.OutputBase}
@param start_sprite_num: Number of the first sprite.
@type start_sprite_num: C{int}
@param compress_grf: Enable GRF sprite compression.
@type compress_grf: C{bool}
@param crop_sprites: Enable sprite cropping.
@type crop_sprites: C{bool}
@param enable_cache: Enable sprite cache.
@type enable_cache: C{bool}
@param forced_palette: Palette to use for the file.
@type forced_palette: C{str}
@param md5_filename: Filename to use for writing the md5 sum of the grf file. C{None} if the file should not be written.
@type md5_filename: C{str} or C{None}
"""
generic.OnlyOnce.clear()
generic.print_progress("Reading ...")
try:
script = inputfile.read()
except UnicodeDecodeError as ex:
raise generic.ScriptError(
'Input file is not utf-8 encoded: {}'.format(ex))
# Strip a possible BOM
script = script.lstrip(str(codecs.BOM_UTF8, "utf-8"))
if script.strip() == "":
generic.print_error("Empty input file")
return 4
generic.print_progress("Init parser ...")
nml_parser = parser.NMLParser(rebuild_parser, debug_parser)
if input_filename is None:
input_filename = 'input'
generic.print_progress("Parsing ...")
result = nml_parser.parse(script, input_filename)
result.validate([])
if output_debug > 0:
result.debug_print(0)
for outputfile in outputfiles:
if isinstance(outputfile, output_nml.OutputNML):
outputfile.open()
outputfile.write(str(result))
outputfile.close()
generic.print_progress("Preprocessing ...")
result.register_names()
result.pre_process()
tmp_actions = result.get_action_list()
generic.print_progress("Generating actions ...")
actions = []
for action in tmp_actions:
if isinstance(action, action1.SpritesetCollection):
actions.extend(action.get_action_list())
else:
actions.append(action)
actions.extend(action11.get_sound_actions())
generic.print_progress("Assigning Action2 registers ...")
action8_index = -1
for i in range(len(actions) - 1, -1, -1):
if isinstance(actions[i],
(action2var.Action2Var, action2layout.Action2Layout)):
actions[i].resolve_tmp_storage()
elif isinstance(actions[i], action8.Action8):
action8_index = i
generic.print_progress("Generating strings ...")
if action8_index != -1:
lang_actions = []
# Add plural/gender/case tables
for lang_pair in grfstrings.langs:
lang_id, lang = lang_pair
lang_actions.extend(action0.get_language_translation_tables(lang))
# Add global strings
lang_actions.extend(action4.get_global_string_actions())
actions = actions[:action8_index +
1] + lang_actions + actions[action8_index + 1:]
generic.print_progress("Collecting real sprites ...")
# Collect all sprite files, and put them into buckets of same image and mask files
sprite_files = dict()
for action in actions:
if isinstance(action, real_sprite.RealSpriteAction):
for sprite in action.sprite_list:
if sprite.is_empty: continue
sprite.validate_size()
file = sprite.file
if file is not None:
file = file.value
mask_file = sprite.mask_file
if mask_file is not None:
mask_file = mask_file.value
key = (file, mask_file)
sprite_files.setdefault(key, []).append(sprite)
# Check whether we can terminate sprite processing prematurely for
# dependency checks
skip_sprite_processing = True
for outputfile in outputfiles:
if isinstance(outputfile, output_dep.OutputDEP):
outputfile.open()
for f in sprite_files:
if f[0] is not None:
outputfile.write(f[0])
if f[1] is not None:
outputfile.write(f[1])
outputfile.close()
skip_sprite_processing &= outputfile.skip_sprite_checks()
if skip_sprite_processing:
generic.clear_progress()
return 0
if not Image and len(sprite_files) > 0:
generic.print_error(
"PIL (python-imaging) wasn't found, no support for using graphics")
sys.exit(3)
generic.print_progress("Checking palette of source images ...")
used_palette = forced_palette
last_file = None
for f_pair in sprite_files:
# Palette is defined by mask_file, if present. Otherwise by the main file.
f = f_pair[1]
if f is None:
f = f_pair[0]
try:
im = Image.open(generic.find_file(f))
except IOError as ex:
raise generic.ImageError(str(ex), f)
if im.mode != "P":
continue
pal = palette.validate_palette(im, f)
if forced_palette != "ANY" and pal != forced_palette and not (
forced_palette == "DEFAULT" and pal == "LEGACY"):
raise generic.ImageError(
"Image has '{}' palette, but you forced the '{}' palette".
format(pal, used_palette), f)
if used_palette == "ANY":
used_palette = pal
elif pal != used_palette:
if used_palette in ("LEGACY", "DEFAULT") and pal in ("LEGACY",
"DEFAULT"):
used_palette = "DEFAULT"
else:
raise generic.ImageError(
"Image has '{}' palette, but \"{}\" has the '{}' palette".
format(pal, last_file, used_palette), f)
last_file = f
palette_bytes = {"LEGACY": "W", "DEFAULT": "D", "ANY": "A"}
if used_palette in palette_bytes:
grf.set_palette_used(palette_bytes[used_palette])
encoder = None
for outputfile in outputfiles:
outputfile.palette = used_palette # used by RecolourSpriteAction
if isinstance(outputfile, output_grf.OutputGRF):
if encoder is None:
encoder = spriteencoder.SpriteEncoder(compress_grf,
crop_sprites,
enable_cache,
used_palette)
outputfile.encoder = encoder
generic.clear_progress()
# Read all image data, compress, and store in sprite cache
if encoder is not None:
encoder.open(sprite_files)
#If there are any 32bpp sprites hint to openttd that we'd like a 32bpp blitter
if alt_sprites.any_32bpp_sprites:
grf.set_preferred_blitter("3")
generic.print_progress("Linking actions ...")
if action8_index != -1:
actions = [sprite_count.SpriteCountAction(len(actions))] + actions
for idx, action in enumerate(actions):
num = start_sprite_num + idx
action.prepare_output(num)
# Processing finished, print some statistics
action0.print_stats()
actionF.print_stats()
action7.print_stats()
action1.print_stats()
action2.print_stats()
action6.print_stats()
grf.print_stats()
global_constants.print_stats()
action4.print_stats()
action11.print_stats()
generic.print_progress("Writing output ...")
md5 = None
for outputfile in outputfiles:
if isinstance(outputfile, output_grf.OutputGRF):
outputfile.open()
for action in actions:
action.write(outputfile)
outputfile.close()
md5 = outputfile.get_md5()
if isinstance(outputfile, output_nfo.OutputNFO):
outputfile.open()
for action in actions:
action.write(outputfile)
outputfile.close()
if md5 is not None and md5_filename is not None:
with open(md5_filename, 'w', encoding="utf-8") as f:
f.write(md5 + '\n')
if encoder is not None:
encoder.close()
generic.clear_progress()
return 0
def main(argv):
global developmode
opts, input_filename = parse_cli(argv)
if opts.stack: developmode = True
grfstrings.read_extra_commands(opts.custom_tags)
generic.print_progress("Reading lang ...")
grfstrings.read_lang_files(opts.lang_dir, opts.default_lang)
generic.clear_progress()
# We have to do the dependency check first or we might later have
# more targets than we asked for
outputs = []
if opts.dep_check:
# First make sure we have a file to output the dependencies to:
dep_filename = opts.dep_filename
if dep_filename is None and opts.grf_filename is not None:
dep_filename = filename_output_from_input(opts.grf_filename,
".dep")
if dep_filename is None and input_filename is not None:
dep_filename = filename_output_from_input(input_filename, ".dep")
if dep_filename is None:
raise generic.ScriptError(
"-M requires a dependency file either via -MF, an input filename or a valid output via --grf"
)
# Now make sure we have a file which is the target for the dependencies:
depgrf_filename = opts.depgrf_filename
if depgrf_filename is None and opts.grf_filename is not None:
depgrf_filename = opts.grf_filename
if depgrf_filename is None and input_filename is not None:
depgrf_filename = filename_output_from_input(
input_filename, ".grf")
if depgrf_filename is None:
raise generic.ScriptError(
"-M requires either a target grf file via -MT, an input filename or a valid output via --grf"
)
# Only append the dependency check to the output targets when we have both,
# a target grf and a file to write to
if dep_filename is not None and depgrf_filename is not None:
outputs.append(output_dep.OutputDEP(dep_filename, depgrf_filename))
if input_filename is None:
input = sys.stdin
else:
input = codecs.open(generic.find_file(input_filename), 'r', 'utf-8')
# Only append an output grf name, if no ouput is given, also not implicitly via -M
if not opts.outputfile_given and not outputs:
opts.grf_filename = filename_output_from_input(
input_filename, ".grf")
# Translate the 'common' palette names as used by OpenTTD to the traditional ones being within NML's code
if opts.forced_palette == 'DOS':
opts.forced_palette = 'DEFAULT'
elif opts.forced_palette == 'WIN':
opts.forced_palette = 'LEGACY'
if opts.grf_filename:
outputs.append(output_grf.OutputGRF(opts.grf_filename))
if opts.nfo_filename:
outputs.append(
output_nfo.OutputNFO(opts.nfo_filename, opts.start_sprite_num))
if opts.nml_filename:
outputs.append(output_nml.OutputNML(opts.nml_filename))
for output in opts.outputs:
outroot, outext = os.path.splitext(output)
outext = outext.lower()
if outext == '.grf': outputs.append(output_grf.OutputGRF(output))
elif outext == '.nfo':
outputs.append(output_nfo.OutputNFO(output, opts.start_sprite_num))
elif outext == '.nml':
outputs.append(output_nml.OutputNML(output))
elif outext == '.dep':
outputs.append(output_dep.OutputDEP(output, opts.grf_filename))
else:
generic.print_error("Unknown output format {}".format(outext))
sys.exit(2)
ret = nml(input, input_filename, opts.debug, outputs,
opts.start_sprite_num, opts.compress, opts.crop,
not opts.no_cache, opts.forced_palette, opts.md5_filename,
opts.rebuild_parser, opts.debug_parser)
input.close()
sys.exit(ret)
def read_cache(self):
"""
Read the *.grf.cache[index] files.
"""
try:
with generic.open_cache_file(self.sources, ".cache", "rb") as cache_file:
cache_data = array.array("B")
cache_size = os.fstat(cache_file.fileno()).st_size
cache_data.fromfile(cache_file, cache_size)
assert cache_size == len(cache_data)
self.cache_time = os.path.getmtime(cache_file.name)
with generic.open_cache_file(self.sources, ".cacheindex", "r") as index_file:
index_file_name = index_file.name
sprite_index = json.load(index_file)
except OSError:
# Cache files don't exist (or otherwise aren't readable)
return
except json.JSONDecodeError:
generic.print_warning(
generic.Warning.GENERIC,
"{} contains invalid data, ignoring.".format(index_file_name)
+ " Please remove the file and file a bug report if this warning keeps appearing",
)
self.cached_sprites = {}
return
source_mtime = {}
try:
# Just assert and print a generic message on errors, as the cache data should be correct
# Not asserting could lead to errors later on
# Also, it doesn't make sense to inform the user about things he shouldn't know about and can't fix
assert isinstance(sprite_index, list)
for sprite in sprite_index:
assert isinstance(sprite, dict)
# load RGB (32bpp) data
rgb_key = (None, None)
if "rgb_file" in sprite and "rgb_rect" in sprite:
assert isinstance(sprite["rgb_file"], str)
assert isinstance(sprite["rgb_rect"], list) and len(sprite["rgb_rect"]) == 4
assert all(isinstance(num, int) for num in sprite["rgb_rect"])
rgb_key = (sprite["rgb_file"], tuple(sprite["rgb_rect"]))
# load Mask (8bpp) data
mask_key = (None, None)
if "mask_file" in sprite and "mask_rect" in sprite:
assert isinstance(sprite["mask_file"], str)
assert isinstance(sprite["mask_rect"], list) and len(sprite["mask_rect"]) == 4
assert all(isinstance(num, int) for num in sprite["mask_rect"])
mask_key = (sprite["mask_file"], tuple(sprite["mask_rect"]))
palette_key = None
if "mask_pal" in sprite:
palette_key = sprite["mask_pal"]
# Compose key
assert any(i is not None for i in rgb_key + mask_key)
key = rgb_key + mask_key + ("crop" in sprite, palette_key)
assert key not in self.cached_sprites
# Read size/offset from cache
assert "offset" in sprite and "size" in sprite
offset, size = sprite["offset"], sprite["size"]
assert isinstance(offset, int) and isinstance(size, int)
assert offset >= 0 and size > 0
assert offset + size <= cache_size
data = cache_data[offset : offset + size]
# Read info / cropping data from cache
assert "info" in sprite and isinstance(sprite["info"], int)
info = sprite["info"]
if "crop" in sprite:
assert isinstance(sprite["crop"], list) and len(sprite["crop"]) == 4
assert all(isinstance(num, int) for num in sprite["crop"])
crop = tuple(sprite["crop"])
else:
crop = None
if "pixel_stats" in sprite:
assert isinstance(sprite["pixel_stats"], dict)
pixel_stats = sprite["pixel_stats"]
else:
pixel_stats = {}
# Compose value
value = (data, info, crop, pixel_stats, True, False)
# Check if cache item is still valid
is_valid = True
if rgb_key[0] is not None:
mtime = source_mtime.get(rgb_key[0])
if mtime is None:
mtime = os.path.getmtime(generic.find_file(rgb_key[0]))
source_mtime[rgb_key[0]] = mtime
if mtime > self.cache_time:
is_valid = False
if mask_key[0] is not None:
mtime = source_mtime.get(mask_key[0])
if mtime is None:
mtime = os.path.getmtime(generic.find_file(mask_key[0]))
source_mtime[mask_key[0]] = mtime
if mtime > self.cache_time:
is_valid = False
# Drop items from older spritecache format without palette entry
if (mask_key[0] is None) != (palette_key is None):
is_valid = False
if is_valid:
self.cached_sprites[key] = value
except Exception:
generic.print_warning(
generic.Warning.GENERIC,
"{} contains invalid data, ignoring.".format(index_file_name)
+ " Please remove the file and file a bug report if this warning keeps appearing",
)
self.cached_sprites = {} # Clear cache
def read_cache(self):
"""
Read the *.grf.cache[index] files.
"""
if not (os.access(self.cache_filename, os.R_OK)
and os.access(self.cache_index_filename, os.R_OK)):
# Cache files don't exist
return
index_file = open(self.cache_index_filename, 'r')
cache_file = open(self.cache_filename, 'rb')
cache_data = array.array('B')
cache_size = os.fstat(cache_file.fileno()).st_size
cache_data.fromfile(cache_file, cache_size)
assert cache_size == len(cache_data)
self.cache_time = os.path.getmtime(self.cache_filename)
source_mtime = dict()
try:
# Just assert and print a generic message on errors, as the cache data should be correct
# Not asserting could lead to errors later on
# Also, it doesn't make sense to inform the user about things he shouldn't know about and can't fix
sprite_index = json.load(index_file)
assert isinstance(sprite_index, list)
for sprite in sprite_index:
assert isinstance(sprite, dict)
# load RGB (32bpp) data
rgb_key = (None, None)
if 'rgb_file' in sprite and 'rgb_rect' in sprite:
assert isinstance(sprite['rgb_file'], str)
assert isinstance(sprite['rgb_rect'], list) and len(
sprite['rgb_rect']) == 4
assert all(
isinstance(num, int) for num in sprite['rgb_rect'])
rgb_key = (sprite['rgb_file'], tuple(sprite['rgb_rect']))
# load Mask (8bpp) data
mask_key = (None, None)
if 'mask_file' in sprite and 'mask_rect' in sprite:
assert isinstance(sprite['mask_file'], str)
assert isinstance(sprite['mask_rect'], list) and len(
sprite['mask_rect']) == 4
assert all(
isinstance(num, int) for num in sprite['mask_rect'])
mask_key = (sprite['mask_file'],
tuple(sprite['mask_rect']))
palette_key = None
if 'mask_pal' in sprite:
palette_key = sprite['mask_pal']
# Compose key
assert any(i is not None for i in rgb_key + mask_key)
key = rgb_key + mask_key + ('crop' in sprite, palette_key)
assert key not in self.cached_sprites
# Read size/offset from cache
assert 'offset' in sprite and 'size' in sprite
offset, size = sprite['offset'], sprite['size']
assert isinstance(offset, int) and isinstance(size, int)
assert offset >= 0 and size > 0
assert offset + size <= cache_size
data = cache_data[offset:offset + size]
# Read info / cropping data from cache
assert 'info' in sprite and isinstance(sprite['info'], int)
info = sprite['info']
if 'crop' in sprite:
assert isinstance(sprite['crop'], list) and len(
sprite['crop']) == 4
assert all(isinstance(num, int) for num in sprite['crop'])
crop = tuple(sprite['crop'])
else:
crop = None
if 'pixel_stats' in sprite:
assert isinstance(sprite['pixel_stats'], dict)
pixel_stats = sprite['pixel_stats']
else:
pixel_stats = {}
# Compose value
value = (data, info, crop, pixel_stats, True, False)
# Check if cache item is still valid
is_valid = True
if rgb_key[0] is not None:
mtime = source_mtime.get(rgb_key[0])
if mtime is None:
mtime = os.path.getmtime(generic.find_file(rgb_key[0]))
source_mtime[rgb_key[0]] = mtime
if mtime > self.cache_time:
is_valid = False
if mask_key[0] is not None:
mtime = source_mtime.get(mask_key[0])
if mtime is None:
mtime = os.path.getmtime(generic.find_file(
mask_key[0]))
source_mtime[mask_key[0]] = mtime
if mtime > self.cache_time:
is_valid = False
# Drop items from older spritecache format without palette entry
if (mask_key[0] is None) != (palette_key is None):
is_valid = False
if is_valid:
self.cached_sprites[key] = value
except:
generic.print_warning(
self.cache_index_filename +
" contains invalid data, ignoring. Please remove the file and file a bug report if this warning keeps appearing"
)
self.cached_sprites = {} # Clear cache
index_file.close()
cache_file.close()
