def main(argv): global developmode opts, input_filename = parse_cli(argv) if opts.stack: developmode = True grfstrings.read_extra_commands(opts.custom_tags) generic.print_progress("Reading lang ...") grfstrings.read_lang_files(opts.lang_dir, opts.default_lang) generic.clear_progress() # We have to do the dependency check first or we might later have # more targets than we asked for outputs = [] if opts.dep_check: # First make sure we have a file to output the dependencies to: dep_filename = opts.dep_filename if dep_filename is None and opts.grf_filename is not None: dep_filename = filename_output_from_input(opts.grf_filename, ".dep") if dep_filename is None and input_filename is not None: dep_filename = filename_output_from_input(input_filename, ".dep") if dep_filename is None: raise generic.ScriptError( "-M requires a dependency file either via -MF, an input filename or a valid output via --grf" ) # Now make sure we have a file which is the target for the dependencies: depgrf_filename = opts.depgrf_filename if depgrf_filename is None and opts.grf_filename is not None: depgrf_filename = opts.grf_filename if depgrf_filename is None and input_filename is not None: depgrf_filename = filename_output_from_input( input_filename, ".grf") if depgrf_filename is None: raise generic.ScriptError( "-M requires either a target grf file via -MT, an input filename or a valid output via --grf" ) # Only append the dependency check to the output targets when we have both, # a target grf and a file to write to if dep_filename is not None and depgrf_filename is not None: outputs.append(output_dep.OutputDEP(dep_filename, depgrf_filename)) if input_filename is None: input = sys.stdin else: input = codecs.open(generic.find_file(input_filename), 'r', 'utf-8') # Only append an output grf name, if no ouput is given, also not implicitly via -M if not opts.outputfile_given and not outputs: opts.grf_filename = filename_output_from_input( input_filename, ".grf") # Translate the 'common' palette names as used by OpenTTD to the traditional ones being within NML's code if opts.forced_palette == 'DOS': opts.forced_palette = 'DEFAULT' elif opts.forced_palette == 'WIN': opts.forced_palette = 'LEGACY' if opts.grf_filename: outputs.append(output_grf.OutputGRF(opts.grf_filename)) if opts.nfo_filename: outputs.append( output_nfo.OutputNFO(opts.nfo_filename, opts.start_sprite_num)) if opts.nml_filename: outputs.append(output_nml.OutputNML(opts.nml_filename)) for output in opts.outputs: outroot, outext = os.path.splitext(output) outext = outext.lower() if outext == '.grf': outputs.append(output_grf.OutputGRF(output)) elif outext == '.nfo': outputs.append(output_nfo.OutputNFO(output, opts.start_sprite_num)) elif outext == '.nml': outputs.append(output_nml.OutputNML(output)) elif outext == '.dep': outputs.append(output_dep.OutputDEP(output, opts.grf_filename)) else: generic.print_error("Unknown output format {}".format(outext)) sys.exit(2) ret = nml(input, input_filename, opts.debug, outputs, opts.start_sprite_num, opts.compress, opts.crop, not opts.no_cache, opts.forced_palette, opts.md5_filename, opts.rebuild_parser, opts.debug_parser) input.close() sys.exit(ret)
def nml(inputfile, input_filename, output_debug, outputfiles, start_sprite_num, compress_grf, crop_sprites, enable_cache, forced_palette, md5_filename, rebuild_parser, debug_parser): """ Compile an NML file. @param inputfile: File handle associated with the input file. @type inputfile: C{File} @param input_filename: Filename of the input file, C{None} if receiving from L{sys.stdin} @type input_filename: C{str} or C{None} @param outputfiles: Output streams to write to. @type outputfiles: C{List} of L{output_base.OutputBase} @param start_sprite_num: Number of the first sprite. @type start_sprite_num: C{int} @param compress_grf: Enable GRF sprite compression. @type compress_grf: C{bool} @param crop_sprites: Enable sprite cropping. @type crop_sprites: C{bool} @param enable_cache: Enable sprite cache. @type enable_cache: C{bool} @param forced_palette: Palette to use for the file. @type forced_palette: C{str} @param md5_filename: Filename to use for writing the md5 sum of the grf file. C{None} if the file should not be written. @type md5_filename: C{str} or C{None} """ generic.OnlyOnce.clear() generic.print_progress("Reading ...") try: script = inputfile.read() except UnicodeDecodeError as ex: raise generic.ScriptError( 'Input file is not utf-8 encoded: {}'.format(ex)) # Strip a possible BOM script = script.lstrip(str(codecs.BOM_UTF8, "utf-8")) if script.strip() == "": generic.print_error("Empty input file") return 4 generic.print_progress("Init parser ...") nml_parser = parser.NMLParser(rebuild_parser, debug_parser) if input_filename is None: input_filename = 'input' generic.print_progress("Parsing ...") result = nml_parser.parse(script, input_filename) result.validate([]) if output_debug > 0: result.debug_print(0) for outputfile in outputfiles: if isinstance(outputfile, output_nml.OutputNML): outputfile.open() outputfile.write(str(result)) outputfile.close() generic.print_progress("Preprocessing ...") result.register_names() result.pre_process() tmp_actions = result.get_action_list() generic.print_progress("Generating actions ...") actions = [] for action in tmp_actions: if isinstance(action, action1.SpritesetCollection): actions.extend(action.get_action_list()) else: actions.append(action) actions.extend(action11.get_sound_actions()) generic.print_progress("Assigning Action2 registers ...") action8_index = -1 for i in range(len(actions) - 1, -1, -1): if isinstance(actions[i], (action2var.Action2Var, action2layout.Action2Layout)): actions[i].resolve_tmp_storage() elif isinstance(actions[i], action8.Action8): action8_index = i generic.print_progress("Generating strings ...") if action8_index != -1: lang_actions = [] # Add plural/gender/case tables for lang_pair in grfstrings.langs: lang_id, lang = lang_pair lang_actions.extend(action0.get_language_translation_tables(lang)) # Add global strings lang_actions.extend(action4.get_global_string_actions()) actions = actions[:action8_index + 1] + lang_actions + actions[action8_index + 1:] generic.print_progress("Collecting real sprites ...") # Collect all sprite files, and put them into buckets of same image and mask files sprite_files = dict() for action in actions: if isinstance(action, real_sprite.RealSpriteAction): for sprite in action.sprite_list: if sprite.is_empty: continue sprite.validate_size() file = sprite.file if file is not None: file = file.value mask_file = sprite.mask_file if mask_file is not None: mask_file = mask_file.value key = (file, mask_file) sprite_files.setdefault(key, []).append(sprite) # Check whether we can terminate sprite processing prematurely for # dependency checks skip_sprite_processing = True for outputfile in outputfiles: if isinstance(outputfile, output_dep.OutputDEP): outputfile.open() for f in sprite_files: if f[0] is not None: outputfile.write(f[0]) if f[1] is not None: outputfile.write(f[1]) outputfile.close() skip_sprite_processing &= outputfile.skip_sprite_checks() if skip_sprite_processing: generic.clear_progress() return 0 if not Image and len(sprite_files) > 0: generic.print_error( "PIL (python-imaging) wasn't found, no support for using graphics") sys.exit(3) generic.print_progress("Checking palette of source images ...") used_palette = forced_palette last_file = None for f_pair in sprite_files: # Palette is defined by mask_file, if present. Otherwise by the main file. f = f_pair[1] if f is None: f = f_pair[0] try: im = Image.open(generic.find_file(f)) except IOError as ex: raise generic.ImageError(str(ex), f) if im.mode != "P": continue pal = palette.validate_palette(im, f) if forced_palette != "ANY" and pal != forced_palette and not ( forced_palette == "DEFAULT" and pal == "LEGACY"): raise generic.ImageError( "Image has '{}' palette, but you forced the '{}' palette". format(pal, used_palette), f) if used_palette == "ANY": used_palette = pal elif pal != used_palette: if used_palette in ("LEGACY", "DEFAULT") and pal in ("LEGACY", "DEFAULT"): used_palette = "DEFAULT" else: raise generic.ImageError( "Image has '{}' palette, but \"{}\" has the '{}' palette". format(pal, last_file, used_palette), f) last_file = f palette_bytes = {"LEGACY": "W", "DEFAULT": "D", "ANY": "A"} if used_palette in palette_bytes: grf.set_palette_used(palette_bytes[used_palette]) encoder = None for outputfile in outputfiles: outputfile.palette = used_palette # used by RecolourSpriteAction if isinstance(outputfile, output_grf.OutputGRF): if encoder is None: encoder = spriteencoder.SpriteEncoder(compress_grf, crop_sprites, enable_cache, used_palette) outputfile.encoder = encoder generic.clear_progress() # Read all image data, compress, and store in sprite cache if encoder is not None: encoder.open(sprite_files) #If there are any 32bpp sprites hint to openttd that we'd like a 32bpp blitter if alt_sprites.any_32bpp_sprites: grf.set_preferred_blitter("3") generic.print_progress("Linking actions ...") if action8_index != -1: actions = [sprite_count.SpriteCountAction(len(actions))] + actions for idx, action in enumerate(actions): num = start_sprite_num + idx action.prepare_output(num) # Processing finished, print some statistics action0.print_stats() actionF.print_stats() action7.print_stats() action1.print_stats() action2.print_stats() action6.print_stats() grf.print_stats() global_constants.print_stats() action4.print_stats() action11.print_stats() generic.print_progress("Writing output ...") md5 = None for outputfile in outputfiles: if isinstance(outputfile, output_grf.OutputGRF): outputfile.open() for action in actions: action.write(outputfile) outputfile.close() md5 = outputfile.get_md5() if isinstance(outputfile, output_nfo.OutputNFO): outputfile.open() for action in actions: action.write(outputfile) outputfile.close() if md5 is not None and md5_filename is not None: with open(md5_filename, 'w', encoding="utf-8") as f: f.write(md5 + '\n') if encoder is not None: encoder.close() generic.clear_progress() return 0
def open(self, sprite_files): """ Start the encoder, read caches, and stuff. @param sprite_files: List of sprites per source image file. @type sprite_files: C{dict} that maps (C{tuple} of C{str}) to (C{RealSprite}) """ num_sprites = sum( len(sprite_list) for sprite_list in sprite_files.values()) generic.print_progress("Encoding ...") num_cached = 0 num_dup = 0 num_enc = 0 num_orphaned = 0 count_sprites = 0 for sources, sprite_list in sprite_files.items(): # Iterate over sprites grouped by source image file. # - Open source files only once. (speed) # - Do not keep files around for long. (memory) source_name = "_".join(src for src in sources if src is not None) local_cache = spritecache.SpriteCache(sources) local_cache.read_cache() for sprite_info in sprite_list: count_sprites += 1 generic.print_progress("Encoding {}/{}: {}".format( count_sprites, num_sprites, source_name), incremental=True) cache_key = sprite_info.get_cache_key(self.crop_sprites) cache_item = local_cache.get_item(cache_key, self.palette) in_use = False in_old_cache = False if cache_item is not None: # Write a sprite from the cached data compressed_data, info_byte, crop_rect, pixel_stats, in_old_cache, in_use = cache_item if in_use: num_dup += 1 else: num_cached += 1 else: ( size_x, size_y, xoffset, yoffset, compressed_data, info_byte, crop_rect, pixel_stats, ) = self.encode_sprite(sprite_info) num_enc += 1 # Store sprite in cache, unless already up-to-date if not in_use: cache_item = (compressed_data, info_byte, crop_rect, pixel_stats, in_old_cache, True) local_cache.add_item(cache_key, self.palette, cache_item) # Delete all files from dictionary to free memory self.cached_image_files.clear() num_orphaned += local_cache.count_orphaned() # Only write cache if compression is enabled. Uncompressed data is not worth to be cached. if self.compress_grf: local_cache.write_cache() # Transfer data to global cache for later usage self.sprite_cache.cached_sprites.update(local_cache.cached_sprites) generic.print_progress("Encoding ...", incremental=True) generic.clear_progress() generic.print_info( "{} sprites, {} cached, {} orphaned, {} duplicates, {} newly encoded ({})" .format(num_sprites, num_cached, num_orphaned, num_dup, num_enc, "native" if lz77.is_native else "python"))