def print_named_filedata(self, filename): name = os.path.split(filename)[1] size = os.path.getsize(filename) self.start_sprite(0, 0xfd) self.sprite_output.start_sprite(8 + 3 + len(name) + 1 + size) self.sprite_output.print_dword(self.sprite_num) self.sprite_output.print_dword(3 + len(name) + 1 + size) self.sprite_output.print_byte(0xff) self.sprite_output.print_byte(0xff) self.sprite_output.print_byte(len(name)) self.print_string( name, force_ascii=True, final_zero=True, stream=self.sprite_output) # ASCII filenames seems sufficient. fp = open(generic.find_file(filename), 'rb') while True: data = fp.read(1024) if len(data) == 0: break for d in data: self.sprite_output.print_byte(d) fp.close() self.sprite_output.end_sprite() self.end_sprite()
def read_extra_commands(custom_tags_file): """ @param custom_tags_file: Filename of the custom tags file. @type custom_tags_file: C{str} """ if not os.access(custom_tags_file, os.R_OK): # Failed to open custom_tags.txt, ignore this return line_no = 0 with open(generic.find_file(custom_tags_file), "r", encoding="utf-8") as fh: for line in fh: line_no += 1 line = line.strip() if len(line) == 0 or line[0] == "#": continue i = line.find(":") if i == -1: raise generic.ScriptError("Line has no ':' delimiter.", generic.LinePosition(custom_tags_file, line_no)) name = line[:i].strip() value = line[i + 1 :] if name in commands: generic.print_warning( 'Overwriting existing tag "' + name + '".', generic.LinePosition(custom_tags_file, line_no) ) commands[name] = {"unicode": value} if is_ascii_string(value): commands[name]["ascii"] = value
def validate_size(self): """ Check if xpos/ypos/xsize/ysize are already set and if not, set them to 0,0,image_width,image_height. """ if self.xpos is None: with Image.open(generic.find_file(self.file.value)) as im: self.xpos = expression.ConstantNumeric(0) self.ypos = expression.ConstantNumeric(0) self.xsize = expression.ConstantNumeric(im.size[0]) self.ysize = expression.ConstantNumeric(im.size[1]) self.check_sprite_size() if self.mask_pos is None: self.mask_pos = (self.xpos, self.ypos)
def open_image_file(self, filename): """ Obtain a handle to an image file @param filename: Name of the file @type filename: C{str} @return: Image file @rtype: L{Image} """ if filename in self.cached_image_files: im = self.cached_image_files[filename] else: im = Image.open(generic.find_file(filename)) self.cached_image_files[filename] = im return im
def parse_file(filename, default): """ Read and parse a single language file. @param filename: The filename of the file to parse. @type filename: C{str} @param default: True iff this is the default language. @type default: C{bool} """ lang = Language(False) try: with open(generic.find_file(filename), "r", encoding="utf-8") as fh: for idx, line in enumerate(fh): pos = generic.LinePosition(filename, idx + 1) line = line.rstrip("\n\r").lstrip("\uFEFF") # The default language is processed twice here. Once as fallback langauge # and once as normal language. if default: default_lang.handle_string(line, pos) lang.handle_string(line, pos) except UnicodeDecodeError: pos = generic.LanguageFilePosition(filename) if default: raise generic.ScriptError("The default language file contains non-utf8 characters.", pos) generic.print_warning("Language file contains non-utf8 characters. Ignoring (part of) the contents.", pos) except generic.ScriptError as err: if default: raise generic.print_warning(err.value, err.pos) else: if lang.langid is None: generic.print_warning( "Language file does not contain a ##grflangid pragma", generic.LanguageFilePosition(filename) ) else: for lng in langs: if lng[0] == lang.langid: msg = "Language file has the same ##grflangid (with number {:d}) as another language file".format( lang.langid ) raise generic.ScriptError(msg, generic.LanguageFilePosition(filename)) langs.append((lang.langid, lang))
def nml(inputfile, input_filename, output_debug, outputfiles, start_sprite_num, compress_grf, crop_sprites, enable_cache, forced_palette, md5_filename, rebuild_parser, debug_parser): """ Compile an NML file. @param inputfile: File handle associated with the input file. @type inputfile: C{File} @param input_filename: Filename of the input file, C{None} if receiving from L{sys.stdin} @type input_filename: C{str} or C{None} @param outputfiles: Output streams to write to. @type outputfiles: C{List} of L{output_base.OutputBase} @param start_sprite_num: Number of the first sprite. @type start_sprite_num: C{int} @param compress_grf: Enable GRF sprite compression. @type compress_grf: C{bool} @param crop_sprites: Enable sprite cropping. @type crop_sprites: C{bool} @param enable_cache: Enable sprite cache. @type enable_cache: C{bool} @param forced_palette: Palette to use for the file. @type forced_palette: C{str} @param md5_filename: Filename to use for writing the md5 sum of the grf file. C{None} if the file should not be written. @type md5_filename: C{str} or C{None} """ generic.OnlyOnce.clear() generic.print_progress("Reading ...") try: script = inputfile.read() except UnicodeDecodeError as ex: raise generic.ScriptError( 'Input file is not utf-8 encoded: {}'.format(ex)) # Strip a possible BOM script = script.lstrip(str(codecs.BOM_UTF8, "utf-8")) if script.strip() == "": generic.print_error("Empty input file") return 4 generic.print_progress("Init parser ...") nml_parser = parser.NMLParser(rebuild_parser, debug_parser) if input_filename is None: input_filename = 'input' generic.print_progress("Parsing ...") result = nml_parser.parse(script, input_filename) result.validate([]) if output_debug > 0: result.debug_print(0) for outputfile in outputfiles: if isinstance(outputfile, output_nml.OutputNML): outputfile.open() outputfile.write(str(result)) outputfile.close() generic.print_progress("Preprocessing ...") result.register_names() result.pre_process() tmp_actions = result.get_action_list() generic.print_progress("Generating actions ...") actions = [] for action in tmp_actions: if isinstance(action, action1.SpritesetCollection): actions.extend(action.get_action_list()) else: actions.append(action) actions.extend(action11.get_sound_actions()) generic.print_progress("Assigning Action2 registers ...") action8_index = -1 for i in range(len(actions) - 1, -1, -1): if isinstance(actions[i], (action2var.Action2Var, action2layout.Action2Layout)): actions[i].resolve_tmp_storage() elif isinstance(actions[i], action8.Action8): action8_index = i generic.print_progress("Generating strings ...") if action8_index != -1: lang_actions = [] # Add plural/gender/case tables for lang_pair in grfstrings.langs: lang_id, lang = lang_pair lang_actions.extend(action0.get_language_translation_tables(lang)) # Add global strings lang_actions.extend(action4.get_global_string_actions()) actions = actions[:action8_index + 1] + lang_actions + actions[action8_index + 1:] generic.print_progress("Collecting real sprites ...") # Collect all sprite files, and put them into buckets of same image and mask files sprite_files = dict() for action in actions: if isinstance(action, real_sprite.RealSpriteAction): for sprite in action.sprite_list: if sprite.is_empty: continue sprite.validate_size() file = sprite.file if file is not None: file = file.value mask_file = sprite.mask_file if mask_file is not None: mask_file = mask_file.value key = (file, mask_file) sprite_files.setdefault(key, []).append(sprite) # Check whether we can terminate sprite processing prematurely for # dependency checks skip_sprite_processing = True for outputfile in outputfiles: if isinstance(outputfile, output_dep.OutputDEP): outputfile.open() for f in sprite_files: if f[0] is not None: outputfile.write(f[0]) if f[1] is not None: outputfile.write(f[1]) outputfile.close() skip_sprite_processing &= outputfile.skip_sprite_checks() if skip_sprite_processing: generic.clear_progress() return 0 if not Image and len(sprite_files) > 0: generic.print_error( "PIL (python-imaging) wasn't found, no support for using graphics") sys.exit(3) generic.print_progress("Checking palette of source images ...") used_palette = forced_palette last_file = None for f_pair in sprite_files: # Palette is defined by mask_file, if present. Otherwise by the main file. f = f_pair[1] if f is None: f = f_pair[0] try: im = Image.open(generic.find_file(f)) except IOError as ex: raise generic.ImageError(str(ex), f) if im.mode != "P": continue pal = palette.validate_palette(im, f) if forced_palette != "ANY" and pal != forced_palette and not ( forced_palette == "DEFAULT" and pal == "LEGACY"): raise generic.ImageError( "Image has '{}' palette, but you forced the '{}' palette". format(pal, used_palette), f) if used_palette == "ANY": used_palette = pal elif pal != used_palette: if used_palette in ("LEGACY", "DEFAULT") and pal in ("LEGACY", "DEFAULT"): used_palette = "DEFAULT" else: raise generic.ImageError( "Image has '{}' palette, but \"{}\" has the '{}' palette". format(pal, last_file, used_palette), f) last_file = f palette_bytes = {"LEGACY": "W", "DEFAULT": "D", "ANY": "A"} if used_palette in palette_bytes: grf.set_palette_used(palette_bytes[used_palette]) encoder = None for outputfile in outputfiles: outputfile.palette = used_palette # used by RecolourSpriteAction if isinstance(outputfile, output_grf.OutputGRF): if encoder is None: encoder = spriteencoder.SpriteEncoder(compress_grf, crop_sprites, enable_cache, used_palette) outputfile.encoder = encoder generic.clear_progress() # Read all image data, compress, and store in sprite cache if encoder is not None: encoder.open(sprite_files) #If there are any 32bpp sprites hint to openttd that we'd like a 32bpp blitter if alt_sprites.any_32bpp_sprites: grf.set_preferred_blitter("3") generic.print_progress("Linking actions ...") if action8_index != -1: actions = [sprite_count.SpriteCountAction(len(actions))] + actions for idx, action in enumerate(actions): num = start_sprite_num + idx action.prepare_output(num) # Processing finished, print some statistics action0.print_stats() actionF.print_stats() action7.print_stats() action1.print_stats() action2.print_stats() action6.print_stats() grf.print_stats() global_constants.print_stats() action4.print_stats() action11.print_stats() generic.print_progress("Writing output ...") md5 = None for outputfile in outputfiles: if isinstance(outputfile, output_grf.OutputGRF): outputfile.open() for action in actions: action.write(outputfile) outputfile.close() md5 = outputfile.get_md5() if isinstance(outputfile, output_nfo.OutputNFO): outputfile.open() for action in actions: action.write(outputfile) outputfile.close() if md5 is not None and md5_filename is not None: with open(md5_filename, 'w', encoding="utf-8") as f: f.write(md5 + '\n') if encoder is not None: encoder.close() generic.clear_progress() return 0
def main(argv): global developmode opts, input_filename = parse_cli(argv) if opts.stack: developmode = True grfstrings.read_extra_commands(opts.custom_tags) generic.print_progress("Reading lang ...") grfstrings.read_lang_files(opts.lang_dir, opts.default_lang) generic.clear_progress() # We have to do the dependency check first or we might later have # more targets than we asked for outputs = [] if opts.dep_check: # First make sure we have a file to output the dependencies to: dep_filename = opts.dep_filename if dep_filename is None and opts.grf_filename is not None: dep_filename = filename_output_from_input(opts.grf_filename, ".dep") if dep_filename is None and input_filename is not None: dep_filename = filename_output_from_input(input_filename, ".dep") if dep_filename is None: raise generic.ScriptError( "-M requires a dependency file either via -MF, an input filename or a valid output via --grf" ) # Now make sure we have a file which is the target for the dependencies: depgrf_filename = opts.depgrf_filename if depgrf_filename is None and opts.grf_filename is not None: depgrf_filename = opts.grf_filename if depgrf_filename is None and input_filename is not None: depgrf_filename = filename_output_from_input( input_filename, ".grf") if depgrf_filename is None: raise generic.ScriptError( "-M requires either a target grf file via -MT, an input filename or a valid output via --grf" ) # Only append the dependency check to the output targets when we have both, # a target grf and a file to write to if dep_filename is not None and depgrf_filename is not None: outputs.append(output_dep.OutputDEP(dep_filename, depgrf_filename)) if input_filename is None: input = sys.stdin else: input = codecs.open(generic.find_file(input_filename), 'r', 'utf-8') # Only append an output grf name, if no ouput is given, also not implicitly via -M if not opts.outputfile_given and not outputs: opts.grf_filename = filename_output_from_input( input_filename, ".grf") # Translate the 'common' palette names as used by OpenTTD to the traditional ones being within NML's code if opts.forced_palette == 'DOS': opts.forced_palette = 'DEFAULT' elif opts.forced_palette == 'WIN': opts.forced_palette = 'LEGACY' if opts.grf_filename: outputs.append(output_grf.OutputGRF(opts.grf_filename)) if opts.nfo_filename: outputs.append( output_nfo.OutputNFO(opts.nfo_filename, opts.start_sprite_num)) if opts.nml_filename: outputs.append(output_nml.OutputNML(opts.nml_filename)) for output in opts.outputs: outroot, outext = os.path.splitext(output) outext = outext.lower() if outext == '.grf': outputs.append(output_grf.OutputGRF(output)) elif outext == '.nfo': outputs.append(output_nfo.OutputNFO(output, opts.start_sprite_num)) elif outext == '.nml': outputs.append(output_nml.OutputNML(output)) elif outext == '.dep': outputs.append(output_dep.OutputDEP(output, opts.grf_filename)) else: generic.print_error("Unknown output format {}".format(outext)) sys.exit(2) ret = nml(input, input_filename, opts.debug, outputs, opts.start_sprite_num, opts.compress, opts.crop, not opts.no_cache, opts.forced_palette, opts.md5_filename, opts.rebuild_parser, opts.debug_parser) input.close() sys.exit(ret)
def read_cache(self): """ Read the *.grf.cache[index] files. """ try: with generic.open_cache_file(self.sources, ".cache", "rb") as cache_file: cache_data = array.array("B") cache_size = os.fstat(cache_file.fileno()).st_size cache_data.fromfile(cache_file, cache_size) assert cache_size == len(cache_data) self.cache_time = os.path.getmtime(cache_file.name) with generic.open_cache_file(self.sources, ".cacheindex", "r") as index_file: index_file_name = index_file.name sprite_index = json.load(index_file) except OSError: # Cache files don't exist (or otherwise aren't readable) return except json.JSONDecodeError: generic.print_warning( generic.Warning.GENERIC, "{} contains invalid data, ignoring.".format(index_file_name) + " Please remove the file and file a bug report if this warning keeps appearing", ) self.cached_sprites = {} return source_mtime = {} try: # Just assert and print a generic message on errors, as the cache data should be correct # Not asserting could lead to errors later on # Also, it doesn't make sense to inform the user about things he shouldn't know about and can't fix assert isinstance(sprite_index, list) for sprite in sprite_index: assert isinstance(sprite, dict) # load RGB (32bpp) data rgb_key = (None, None) if "rgb_file" in sprite and "rgb_rect" in sprite: assert isinstance(sprite["rgb_file"], str) assert isinstance(sprite["rgb_rect"], list) and len(sprite["rgb_rect"]) == 4 assert all(isinstance(num, int) for num in sprite["rgb_rect"]) rgb_key = (sprite["rgb_file"], tuple(sprite["rgb_rect"])) # load Mask (8bpp) data mask_key = (None, None) if "mask_file" in sprite and "mask_rect" in sprite: assert isinstance(sprite["mask_file"], str) assert isinstance(sprite["mask_rect"], list) and len(sprite["mask_rect"]) == 4 assert all(isinstance(num, int) for num in sprite["mask_rect"]) mask_key = (sprite["mask_file"], tuple(sprite["mask_rect"])) palette_key = None if "mask_pal" in sprite: palette_key = sprite["mask_pal"] # Compose key assert any(i is not None for i in rgb_key + mask_key) key = rgb_key + mask_key + ("crop" in sprite, palette_key) assert key not in self.cached_sprites # Read size/offset from cache assert "offset" in sprite and "size" in sprite offset, size = sprite["offset"], sprite["size"] assert isinstance(offset, int) and isinstance(size, int) assert offset >= 0 and size > 0 assert offset + size <= cache_size data = cache_data[offset : offset + size] # Read info / cropping data from cache assert "info" in sprite and isinstance(sprite["info"], int) info = sprite["info"] if "crop" in sprite: assert isinstance(sprite["crop"], list) and len(sprite["crop"]) == 4 assert all(isinstance(num, int) for num in sprite["crop"]) crop = tuple(sprite["crop"]) else: crop = None if "pixel_stats" in sprite: assert isinstance(sprite["pixel_stats"], dict) pixel_stats = sprite["pixel_stats"] else: pixel_stats = {} # Compose value value = (data, info, crop, pixel_stats, True, False) # Check if cache item is still valid is_valid = True if rgb_key[0] is not None: mtime = source_mtime.get(rgb_key[0]) if mtime is None: mtime = os.path.getmtime(generic.find_file(rgb_key[0])) source_mtime[rgb_key[0]] = mtime if mtime > self.cache_time: is_valid = False if mask_key[0] is not None: mtime = source_mtime.get(mask_key[0]) if mtime is None: mtime = os.path.getmtime(generic.find_file(mask_key[0])) source_mtime[mask_key[0]] = mtime if mtime > self.cache_time: is_valid = False # Drop items from older spritecache format without palette entry if (mask_key[0] is None) != (palette_key is None): is_valid = False if is_valid: self.cached_sprites[key] = value except Exception: generic.print_warning( generic.Warning.GENERIC, "{} contains invalid data, ignoring.".format(index_file_name) + " Please remove the file and file a bug report if this warning keeps appearing", ) self.cached_sprites = {} # Clear cache
def read_cache(self): """ Read the *.grf.cache[index] files. """ if not (os.access(self.cache_filename, os.R_OK) and os.access(self.cache_index_filename, os.R_OK)): # Cache files don't exist return index_file = open(self.cache_index_filename, 'r') cache_file = open(self.cache_filename, 'rb') cache_data = array.array('B') cache_size = os.fstat(cache_file.fileno()).st_size cache_data.fromfile(cache_file, cache_size) assert cache_size == len(cache_data) self.cache_time = os.path.getmtime(self.cache_filename) source_mtime = dict() try: # Just assert and print a generic message on errors, as the cache data should be correct # Not asserting could lead to errors later on # Also, it doesn't make sense to inform the user about things he shouldn't know about and can't fix sprite_index = json.load(index_file) assert isinstance(sprite_index, list) for sprite in sprite_index: assert isinstance(sprite, dict) # load RGB (32bpp) data rgb_key = (None, None) if 'rgb_file' in sprite and 'rgb_rect' in sprite: assert isinstance(sprite['rgb_file'], str) assert isinstance(sprite['rgb_rect'], list) and len( sprite['rgb_rect']) == 4 assert all( isinstance(num, int) for num in sprite['rgb_rect']) rgb_key = (sprite['rgb_file'], tuple(sprite['rgb_rect'])) # load Mask (8bpp) data mask_key = (None, None) if 'mask_file' in sprite and 'mask_rect' in sprite: assert isinstance(sprite['mask_file'], str) assert isinstance(sprite['mask_rect'], list) and len( sprite['mask_rect']) == 4 assert all( isinstance(num, int) for num in sprite['mask_rect']) mask_key = (sprite['mask_file'], tuple(sprite['mask_rect'])) palette_key = None if 'mask_pal' in sprite: palette_key = sprite['mask_pal'] # Compose key assert any(i is not None for i in rgb_key + mask_key) key = rgb_key + mask_key + ('crop' in sprite, palette_key) assert key not in self.cached_sprites # Read size/offset from cache assert 'offset' in sprite and 'size' in sprite offset, size = sprite['offset'], sprite['size'] assert isinstance(offset, int) and isinstance(size, int) assert offset >= 0 and size > 0 assert offset + size <= cache_size data = cache_data[offset:offset + size] # Read info / cropping data from cache assert 'info' in sprite and isinstance(sprite['info'], int) info = sprite['info'] if 'crop' in sprite: assert isinstance(sprite['crop'], list) and len( sprite['crop']) == 4 assert all(isinstance(num, int) for num in sprite['crop']) crop = tuple(sprite['crop']) else: crop = None if 'pixel_stats' in sprite: assert isinstance(sprite['pixel_stats'], dict) pixel_stats = sprite['pixel_stats'] else: pixel_stats = {} # Compose value value = (data, info, crop, pixel_stats, True, False) # Check if cache item is still valid is_valid = True if rgb_key[0] is not None: mtime = source_mtime.get(rgb_key[0]) if mtime is None: mtime = os.path.getmtime(generic.find_file(rgb_key[0])) source_mtime[rgb_key[0]] = mtime if mtime > self.cache_time: is_valid = False if mask_key[0] is not None: mtime = source_mtime.get(mask_key[0]) if mtime is None: mtime = os.path.getmtime(generic.find_file( mask_key[0])) source_mtime[mask_key[0]] = mtime if mtime > self.cache_time: is_valid = False # Drop items from older spritecache format without palette entry if (mask_key[0] is None) != (palette_key is None): is_valid = False if is_valid: self.cached_sprites[key] = value except: generic.print_warning( self.cache_index_filename + " contains invalid data, ignoring. Please remove the file and file a bug report if this warning keeps appearing" ) self.cached_sprites = {} # Clear cache index_file.close() cache_file.close()