def _minimal_valid_ff(num_cols, num_rows, num_levels, start_lon, start_lat, col_spacing, row_spacing, grid_stagger): """ Return a basic field-file object; populating the bare minimum header inputs for validation. """ ff = mule.FieldsFile() ff.fixed_length_header.dataset_type = 3 ff.fixed_length_header.grid_staggering = grid_stagger ff.integer_constants = mule.ff.FF_IntegerConstants.empty() ff.integer_constants.num_cols = num_cols ff.integer_constants.num_rows = num_rows ff.integer_constants.num_p_levels = num_levels ff.real_constants = mule.ff.FF_RealConstants.empty() ff.real_constants.start_lon = start_lon ff.real_constants.start_lat = start_lat ff.real_constants.col_spacing = col_spacing ff.real_constants.row_spacing = row_spacing ff.level_dependent_constants = ( mule.ff.FF_LevelDependentConstants.empty(num_levels + 1)) ldc_range = np.arange(num_levels + 1) for idim in range(1, ff.level_dependent_constants.shape[1] + 1): ff.level_dependent_constants.raw[:, idim] = ldc_range*idim stashmaster = STASHmaster.from_file(SAMPLE_STASHMASTER) ff.stashmaster = stashmaster return ff
def _main(): """ Main function; accepts command line argument for paths to the input and output files. """ # Setup help text help_prolog = """ usage: %(prog)s [-h] input_filename output_filename This script will write a new file where all WGDOS packed fields in the original are replaced by unpacked fields. """ title = _banner( "UNPACK - Unpacks WGDOS packed fields in a UM FieldsFile " "(Using the Mule API)", banner_char="=") parser = argparse.ArgumentParser( usage=argparse.SUPPRESS, description=title + textwrap.dedent(help_prolog), formatter_class=argparse.RawTextHelpFormatter, ) # No need to output help text for the input files (it's obvious) parser.add_argument("input_filename", help=argparse.SUPPRESS) parser.add_argument("output_filename", help=argparse.SUPPRESS) # If the user supplied no arguments, print the help text and exit if len(sys.argv) == 1: parser.print_help() parser.exit(1) args = parser.parse_args() # Print version information print(_banner("(UNPACK) Module Information")), report_modules() print("") input_filename = args.input_filename if not os.path.exists(input_filename): msg = "File not found: {0}".format(input_filename) raise ValueError(msg) output_filename = args.output_filename # Check if the file is a pp file pp_mode = mule.pp.file_is_pp_file(input_filename) if pp_mode: # Make an empty fieldsfile object and attach the pp file's # field objects to it origfile = mule.FieldsFile() origfile.fields = mule.pp.fields_from_pp_file(input_filename) origfile._source_path = input_filename else: # Read in file as a FieldsFile origfile = mule.FieldsFile.from_file(input_filename) # Unpack fieldsfile unpackedfile = unpack(origfile) # Write file if pp_mode: mule.pp.fields_to_pp_file(output_filename, unpackedfile.fields) else: unpackedfile.to_file(output_filename)
if key.startswith("lb"): new_vals.append(int(val)) else: new_vals.append(float(val)) entry[inc_exc][key] = new_vals if entry: case_dicts.append(entry) # Load the input files selected_fields = [] pp_mode = False for ifile, input_file in enumerate(input_files): # Check if this is a pp file if mule.pp.file_is_pp_file(input_file): pp_mode = True umf = mule.FieldsFile() umf.fields = mule.pp.fields_from_pp_file(input_file) umf._source_path = input_file elif not pp_mode: umf = mule.load_umfile(input_file) else: msg = "Cannot mix and match UM files and pp files" raise ValueError(msg) # Iterate through the cases - each returns a list of matching fields for case in case_dicts: selected_fields += (select(umf, include=case.get("include", None), exclude=case.get("exclude", None))) # Copy first file to use for writing output
def gen_pert_file(umf_clim, alpha, ens_member, date): """ Generate an SST perturbation file from an input file containing a set of climatological fields and some values to setup a random number generator. Args: * umf_clim: A :class:`mule.UMFile` object containing the 12 field objects giving the SST (lbuser4=24) for each month of the year. * alpha: Factor used by algorithm (higher values lead to more extreme perturbations). * ens_member: Ensemble member number - used in random generator. * date: Datetime object giving the desired date for the perturbed field. Returns: * umf_pert: A :class:`mule.FieldsFile` object containing the perturbed SST field. """ # Generate the perturbations; assume that the first 12 fields # in the file are the SST fields pert_field = gen_pert_field(umf_clim.fields[:12], alpha, ens_member, date) # Create a FieldsFile object for the output file ff = mule.FieldsFile() # Copy the fixed length header from the input, changing the type # to match a FieldsFile ff.fixed_length_header = umf_clim.fixed_length_header ff.fixed_length_header.dataset_type = 3 # Populate the time entries from the date given now = datetime.now() ff.fixed_length_header.t1_year = date.year ff.fixed_length_header.t2_year = date.year ff.fixed_length_header.t3_year = now.year ff.fixed_length_header.t1_month = date.month ff.fixed_length_header.t2_month = date.month ff.fixed_length_header.t3_month = now.month ff.fixed_length_header.t1_day = date.day ff.fixed_length_header.t2_day = date.day ff.fixed_length_header.t3_day = now.day ff.fixed_length_header.t1_hour = date.hour ff.fixed_length_header.t2_hour = date.hour ff.fixed_length_header.t3_hour = now.hour ff.fixed_length_header.t1_minute = date.minute ff.fixed_length_header.t2_minute = date.minute ff.fixed_length_header.t3_minute = now.minute ff.fixed_length_header.t1_second = date.second ff.fixed_length_header.t2_second = date.second ff.fixed_length_header.t3_second = now.second # Copy the integer headers that are populated ff.integer_constants = mule.ff.FF_IntegerConstants.empty() ff.integer_constants.raw[:len(umf_clim.integer_constants.raw)] = ( umf_clim.integer_constants.raw) # Copy the real headers that are populated ff.real_constants = mule.ff.FF_RealConstants.empty() ff.real_constants.raw[:len(umf_clim.real_constants.raw)] = ( umf_clim.real_constants.raw) # We can't be sure that the input file's grid was setup sensibly, so # assume that the copied field header is right and setup the file # headers from that ff.fixed_length_header.grid_staggering = 6 ff.real_constants.start_lat = pert_field.bzy + 0.5 * pert_field.bdy ff.real_constants.start_lon = pert_field.bzx + 0.5 * pert_field.bdx # The ancil is likely missing the level dependent constants, so # put some in which are the minimum size for a FieldsFile if umf_clim.level_dependent_constants is None: ff.integer_constants.num_p_levels = 1 ff.level_dependent_constants = ( mule.ff.FF_LevelDependentConstants.empty(2)) # Copy the row/column headers if they were set if (umf_clim.row_dependent_constants is not None and umf_clim.column_dependent_constants is not None): ff.row_dependent_constants = umf_clim.row_dependent_constants ff.column_dependent_constants = umf_clim.column_dependent_constants # Add the perturbation field and output ff.fields = [pert_field] return ff
def _main(): """ Main function; accepts command line arguments to override the print settings and provides a UM file to summarise. """ # Setup help text help_prolog = """ usage: %(prog)s [-h] [options] input_file This script will output a summary table of the lookup headers in a UM file, with the columns selected by the user. """ title = _banner( "SUMMARY - Print a summary of the fields in a UM File " "(using the Mule API)", banner_char="=") # Include a list of the lookup names as they appear in Mule lookup_names = [name for name, _ in mule._LOOKUP_HEADER_3] lookup_names += [ name for name, _ in mule._LOOKUP_HEADER_2 if name not in lookup_names ] lookup_names = ", ".join(lookup_names) help_epilog = """ possible lookup names for the column-names option: {0} for details of how these relate to indices see UMDP F03: https://code.metoffice.gov.uk/doc/um/latest/papers/umdp_F03.pdf """.format( textwrap.fill(lookup_names, width=80, initial_indent=4 * " ", subsequent_indent=8 * " ")) # Setup the parser parser = argparse.ArgumentParser( usage=argparse.SUPPRESS, description=title + textwrap.dedent(help_prolog), epilog=textwrap.dedent(help_epilog), formatter_class=argparse.RawTextHelpFormatter, ) # No need to output help text for hte input file (it's obvious) parser.add_argument("input_file", help=argparse.SUPPRESS) parser.add_argument( "--column-names", metavar="--column-names name1[,name2][...]", help="set the names of the lookup header items to print, in the \n" "order the columns should appear as a comma separated list. A \n" "special entry of \"stash_name\" will put in the field's name \n" "according to the STASHmaster, \"index\" will give the field's \n" "index number in the file, and \"t1\" or \"t2\" will give the first \n" "and second time from the lookup (nicely formatted)\n ") parser.add_argument( "--heading-frequency", metavar="N", type=int, help="repeat the column heading block every N lines (to avoid \n" "having to scroll too far to identify columns in the output) A \n" "value of 0 means do not repeat the heading block\n ") parser.add_argument( "--field-index", metavar="i1[,i2][,i3:i5][...]", help="limit the output to specific fields by index (comma-separated \n" "list of single indices, or ranges of indices separated by a single \n" "colon-character)\n ") parser.add_argument( "--field-property", metavar="key1=value1[,key2=value2][...]", help="limit the output to specific fields using a property string \n" "(comma-separated list of key=value pairs where key is the name of \n" "a lookup property and value is what it must be set to)\n ") parser.add_argument( "--stashmaster", help="either the full path to a valid stashmaster file, or a UM \n" "version number e.g. '10.2'; if given a number summary will look in \n" "the path defined by: \n" " mule.stashmaster.STASHMASTER_PATH_PATTERN \n" "which by default is : \n" " $UMDIR/vnX.X/ctldata/STASHmaster/STASHmaster_A\n") # If the user supplied no arguments, print the help text and exit if len(sys.argv) == 1: parser.print_help() parser.exit(1) args = parser.parse_args() # Print version information print(_banner("(SUMMARY) Module Information")), report_modules() print("") # Process column names if args.column_names is not None: properties = args.column_names.split(",") PRINT_SETTINGS["column_names"] = properties # Process field filtering by index argument field_index = [] if args.field_index is not None: for arg in args.field_index.split(","): if re.match(r"^\d+$", arg): field_index.append(int(arg)) elif re.match(r"^\d+:\d+$", arg): field_index += range(*[int(elt) for elt in arg.split(":")]) else: msg = "Unrecognised field index option: {0}" raise ValueError(msg.format(arg)) PRINT_SETTINGS["field_index"] = field_index # Process field filtering by property argument field_property = {} if args.field_property is not None: for arg in args.field_property.split(","): if re.match(r"^\w+=\d+$", arg): name, value = arg.split("=") field_property[name] = int(value) else: msg = "Unrecognised field property option: {0}" raise ValueError(msg.format(arg)) PRINT_SETTINGS["field_property"] = field_property # If provided, load the given stashmaster stashm = None if args.stashmaster is not None: if re.match(r"\d+.\d+", args.stashmaster): stashm = STASHmaster.from_version(args.stashmaster) else: stashm = STASHmaster.from_file(args.stashmaster) if stashm is None: msg = "Cannot load user supplied STASHmaster" raise ValueError(msg) # Process heading lines if args.heading_frequency is not None: PRINT_SETTINGS["heading_frequency"] = args.heading_frequency # Get the filename and load it using Mule filename = args.input_file if os.path.exists(filename): # Check if this is a pp file if mule.pp.file_is_pp_file(filename): # Make an empty fieldsfile object and attach the pp file's # field objects to it um_file = mule.FieldsFile() um_file.fields = mule.pp.fields_from_pp_file(filename) um_file._source_path = filename if stashm is not None: um_file.attach_stashmaster_info(stashm) # Override the component filter as only the lookup is # available in a pp file PRINT_SETTINGS["component_filter"] = ["lookup"] else: um_file = mule.load_umfile(filename, stashmaster=stashm) # Now print the object to stdout, if a SIGPIPE is received handle # it appropriately try: field_summary(um_file) except IOError as error: if error.errno != errno.EPIPE: raise else: msg = "File not found: {0}".format(filename) raise ValueError(msg)
def _main(): """ Main function; accepts command line arguments to override the print settings and provides a UM file to print. """ # Setup help text help_prolog = """ usage: %(prog)s [-h] [options] input_filename This script will output the contents of the headers from a UM file to stdout. The default output may be customised with a variety of options (see below). """ title = _banner( "PUMF-II - Pretty Printer for UM Files, version II " "(using the Mule API)", banner_char="=") # Include a list of the component names as they appear in Mule component_names = ", ".join( (["fixed_length_header"] + [name for name, _ in mule.UMFile.COMPONENTS] + ["lookup"])) lookup_names = [name for name, _ in mule._LOOKUP_HEADER_3] lookup_names += [ name for name, _ in mule._LOOKUP_HEADER_2 if name not in lookup_names ] lookup_names = ", ".join(lookup_names) help_epilog = """ possible component names for the component option: {0} possible lookup names for the field-property option: {1} for details of how these relate to indices see UMDP F03: https://code.metoffice.gov.uk/doc/um/latest/papers/umdp_F03.pdf """.format( textwrap.fill(component_names, width=80, initial_indent=4 * " ", subsequent_indent=8 * " "), textwrap.fill(lookup_names, width=80, initial_indent=4 * " ", subsequent_indent=8 * " ")) # Setup the parser parser = argparse.ArgumentParser( usage=argparse.SUPPRESS, description=title + textwrap.dedent(help_prolog), epilog=textwrap.dedent(help_epilog), formatter_class=argparse.RawTextHelpFormatter, ) # No need to output help text for the input file (it's obvious) parser.add_argument("input_file", help=argparse.SUPPRESS) parser.add_argument( "--include-missing", help="include header values which are set to MDI and entries for \n" "components which are not present in the file (by default this will \n" "be hidden)\n ", action="store_true") parser.add_argument( "--use-indices", help="list headers by their indices (instead of only listing named \n" "headers)\n ", action="store_true") parser.add_argument( "--headers-only", help="only list headers (do not read data and calculate any derived \n" "statistics)\n", action="store_true") parser.add_argument( "--components", help="limit the header output to specific components \n" "(comma-separated list of component names, with no spaces)\n ", metavar="component1[,component2][...]") parser.add_argument( "--field-index", help="limit the lookup output to specific fields by index \n" "(comma-separated list of single indices, or ranges of indices \n" "separated by a single colon-character)\n ", metavar="i1[,i2][,i3:i5][...]") parser.add_argument( "--field-property", help="limit the lookup output to specific field using a property \n" "string (comma-separated list of key=value pairs where the key is \n" "the name of a lookup property and the value is the value it must \n" "take)\n ", metavar="key1=value1[,key2=value2][...]") parser.add_argument("--print-columns", help="how many columns should be printed\n ", metavar="N") parser.add_argument( "--stashmaster", help="either the full path to a valid stashmaster file, or a UM \n" "version number e.g. '10.2'; if given a number pumf will look in \n" "the path defined by: \n" " mule.stashmaster.STASHMASTER_PATH_PATTERN \n" "which by default is : \n" " $UMDIR/vnX.X/ctldata/STASHmaster/STASHmaster_A\n") # If the user supplied no arguments, print the help text and exit if len(sys.argv) == 1: parser.print_help() parser.exit(1) args = parser.parse_args() # Print version information print(_banner("(PUMF-II) Module Information")), report_modules() print("") # Process component filtering argument if args.components is not None: PRINT_SETTINGS["component_filter"] = (args.components.split(",")) # Process field filtering by index argument field_index = [] if args.field_index is not None: for arg in args.field_index.split(","): if re.match(r"^\d+$", arg): field_index.append(int(arg)) elif re.match(r"^\d+:\d+$", arg): field_index += range(*[int(elt) for elt in arg.split(":")]) else: msg = "Unrecognised field-index option: {0}" raise ValueError(msg.format(arg)) PRINT_SETTINGS["field_index"] = field_index # Process field filtering by property argument field_property = {} if args.field_property is not None: for arg in args.field_property.split(","): if re.match(r"^\w+=\d+$", arg): name, value = arg.split("=") field_property[name] = int(value) else: msg = "Unrecognised field-property option: {0}" raise ValueError(msg.format(arg)) PRINT_SETTINGS["field_property"] = field_property # If provided, load the given stashmaster stashm = None if args.stashmaster is not None: if re.match(r"\d+.\d+", args.stashmaster): stashm = STASHmaster.from_version(args.stashmaster) else: stashm = STASHmaster.from_file(args.stashmaster) if stashm is None: msg = "Cannot load user supplied STASHmaster" raise ValueError(msg) # Process remaining options if args.print_columns is not None: PRINT_SETTINGS["print_columns"] = int(args.print_columns) if args.include_missing: PRINT_SETTINGS["include_missing"] = True if args.use_indices: PRINT_SETTINGS["use_indices"] = True if args.headers_only: PRINT_SETTINGS["headers_only"] = True # Get the filename and load it using Mule filename = args.input_file if os.path.exists(filename): # Check if this is a pp file if mule.pp.file_is_pp_file(filename): # Make an empty fieldsfile object and attach the pp file's # field objects to it um_file = mule.FieldsFile() um_file.fields = mule.pp.fields_from_pp_file(filename) um_file._source_path = filename if stashm is not None: um_file.attach_stashmaster_info(stashm) # Override the component filter as only the lookup is # available in a pp file PRINT_SETTINGS["component_filter"] = ["lookup"] else: um_file = mule.load_umfile(filename, stashmaster=stashm) # Now print the object to stdout, if a SIGPIPE is received handle # it appropriately try: pprint(um_file) except IOError as error: if error.errno != errno.EPIPE: raise else: msg = "File not found: {0}".format(filename) raise ValueError(msg)