def equalise_data_type(cubes, data_type='float32'): """ Casts datatypes in iris numpy array to be of the same datatype. Args: cubes: Cubes to have their datatypes equalised. data_type: String specifying datatype, default is float32. Returns: cubes: Cubes with their data types identical. """ logger = log_module() if data_type == 'float32': for cube in cubes: cube.data = np.float32(cube.data) elif data_type == 'float64': for cube in cubes: cube.data = np.float64(cube.data) elif data_type == 'int32': for cube in cubes: cube.data = np.int32(cube.data) elif data_type == 'int64': for cube in cubes: cube.data = np.int64(cube.data) else: logger.error("invalid data type")
def test_log_module_singleton(self): logger = log_module() looger_a_keys = list(logger.__dict__.keys()) looger_b_keys = list(logger.__dict__.keys()) looger_a_vals = list(logger.__dict__.values()) looger_b_vals = list(logger.__dict__.values()) self.assertEqual(looger_a_keys, looger_b_keys) self.assertEqual(looger_a_vals, looger_b_vals)
def test_reset_logger(self): reset_logger() logger = log_module() handler = logger.handlers[0] self.assertEqual(len(logger.handlers), 1) self.assertEqual(logger.name, 'cube_helper.logger') self.assertEqual(logger.level, 20) self.assertEqual(handler.level, 20) self.assertEqual(handler.stream, sys.stdout)
def test_log_module_redirect(self): logger = log_module() out = IO() with _redirect_stdout(out): logger.info('Message on stdout') output = out.getvalue().strip() self.assertEqual(output, 'Message on stdout') out = IO() with _redirect_stdout(out): logger.info('Message on stderr') output = out.getvalue().strip() self.assertEqual(output, 'Message on stderr')
def load(directory, filetype='.nc', constraints=None): """ A function that loads and concatenates Iris Cubes. Args: directory: A String specifying the directory or filename of the Cubes you wish to concatenate. Accepts either directory location or a list or glob object of individual Cubes. filetype: Extension of Iris Cubes to Load. set to '.nc' by default. constraints: Any constraints to be applied to Cubes on load. Returns: result: A concatenated Iris Cube. """ logger = log_module() if isinstance(directory, string_types): loaded_cubes, cube_files = load_from_dir(directory, filetype, constraints) if not loaded_cubes: raise OSError("No cubes loaded") else: compare_cubes(loaded_cubes) result = equalise_all(loaded_cubes) result = iris.cube.CubeList(result) try: result = result.concatenate_cube() return result except iris.exceptions.ConcatenateError: logger.info("\nThere was an error in concatenation\n") err_msg = _examine_dim_bounds(result, cube_files) logger.error(err_msg) raise elif isinstance(directory, list): loaded_cubes, cube_files = load_from_filelist(directory, filetype, constraints) if not loaded_cubes: raise OSError("No cubes loaded") else: compare_cubes(loaded_cubes) result = equalise_all(loaded_cubes) result = iris.cube.CubeList(result) try: result = result.concatenate_cube() return result except iris.exceptions.ConcatenateError: logger.info("\nThere was an error in concatenation\n") err_msg = _examine_dim_bounds(result, cube_files) logger.error(err_msg) raise
def equalise_aux_coords(cubes, comp_only=False): """ Equalises auxillary coordinates of cubes. Args: cubes: CubeList or list of Cubes to equalise. comp_only: A boolean value, if set to True it will examine the Cube's aux coordinates and print inconsistencies but not equalise them. Returns: Cubes equalised across auxillary coordinates. """ logger = log_module() inconsistencies = set({}) change_messages = set({}) cube_combs = list(combinations(cubes, 2)) for combs in cube_combs: cube_a_dict = {c.name(): c for c in combs[0].aux_coords} cube_b_dict = {c.name(): c for c in combs[1].aux_coords} cube_a_coords = {c for c in cube_a_dict} cube_b_coords = {c for c in cube_b_dict} for coord in list(cube_a_coords): if coord not in cube_b_coords: if comp_only: inconsistencies.add(coord) elif coord == 'height': change_messages.add("Adding {} coords to cube\n". format(coord)) combs[1].add_aux_coord(cube_a_dict[coord]) for coord in list(cube_b_coords): if coord not in cube_a_coords: if comp_only: inconsistencies.add(coord) elif coord == 'height': change_messages.add("Adding {} coords to cube\n". format(coord)) combs[0].add_aux_coord(cube_b_dict[coord]) if inconsistencies: inconsistencies = list(inconsistencies) log_inconsistent(inconsistencies, 'coords') if change_messages: for message in change_messages: logger.info(message) return cubes
def examine_dim_bounds(cubes, cube_files): """ Examines the dimensional bounds of time should concatenate fail. Cycles through cubes and determines if the times are contiguous. Args: cubes: Iris cubes to examine the time bounds of cube_files: the respective files of cubes, to give users info as to what cubes are causing problems with concatentation. Returns: A printed string detailing any overlap in the time bounds. """ logger = log_module() msg = _examine_dim_bounds(cubes, cube_files) logger.info(msg)
def equalise_time_units(cubes, comp_only=False): """ Equalises time units by cycling through each cube in the given CubeList or list of loaded cubes. Args: cubes: Cubes to equalised of time coords. comp_only: A boolean value, if set to True it will examine the cubes time_coordinates and print inconsistencies but not equalise them. Returns: cubes with time coordinates unified. """ logger = log_module() comp_messages = set() change_messages = set() calendar = cubes[0].coord('time').units.calendar origin = cubes[0].coord('time').units.origin for cube in cubes: for time_coord in cube.coords(): if time_coord.units.is_time_reference(): if comp_only: if time_coord.units.calendar != calendar: comp_messages.add("\tcalendar format inconsistent\n") if time_coord.units.origin != origin: comp_messages.add("\ttime start date inconsistent\n") else: if origin != time_coord.units.origin: change_messages.add("New time origin set to " "{}\n".format(origin)) unify_time_units(cubes) if comp_messages: for message in comp_messages: logger.info(message) if change_messages: for message in change_messages: logger.info(message) return cubes
def compare_cubes(cubes): """ Examines coordinates and attributes across iterable of iris cubes And calls equalise functions (with comp_only arg set to true) where appropriate. Args: cubes: An iterable of iris Cubes or CubeList to be compared for inconsostencies. Returns: A printed string detailing the inconsistencies in the cubes. """ logger = log_module() uneq_aux_coords = False uneq_dim_coords = False uneq_attr = False uneq_time_coords = False uneq_ndim = False cube_combs = list(combinations(cubes, 2)) for comb in cube_combs: if (uneq_aux_coords and uneq_attr and uneq_dim_coords and uneq_time_coords and uneq_ndim): break if comb[0].aux_coords != comb[1].aux_coords: uneq_aux_coords = True if comb[0].dim_coords != comb[1].dim_coords: uneq_dim_coords = True if comb[0].attributes != comb[1].attributes: uneq_attr = True if comb[0].ndim != comb[1].ndim: uneq_ndim = True break if comb[0].coord('time').units != comb[1].coord('time').units: uneq_time_coords = True break if uneq_ndim: logger.error("Number of dimensions for cubes differ," " please load cubes of matching ndim") raise OSError if uneq_aux_coords: logger.info("\ncube aux coordinates differ: \n") equalise_aux_coords(cubes, comp_only=True) if uneq_dim_coords: equalise_dim_coords(cubes, comp_only=True) if uneq_attr: logger.info("cube attributes differ: \n") equalise_attributes(cubes, comp_only=True) if uneq_time_coords: logger.info("cube time coordinates differ: \n") equalise_time_units(cubes, comp_only=True)
def equalise_dim_coords(cubes, comp_only=False): """ Equalises dimensional coordinates of Cubes, specifically long_name, standard_name, and var_name. Args: cubes: CubeList or list of Cubes to equalise. comp_only: A boolean value, if set to True it will examine the cubes dimension coordinates and print inconsistencies but not equalise them. Returns: Cubes equalised across dimension coordinates. """ logger = log_module() inconsistency_sn = set() inconsistency_ln = set() inconsistency_vn = set() inconsistency_attr = set() coord_dict = {} for cube in cubes: for coord in cube.dim_coords: coord_dict.update( {coord.name(): {'long_name': coord.long_name, 'standard_name': coord.standard_name, 'var_name': coord.var_name, 'attributes': coord.attributes}}) for coord in coord_dict: for cube in cubes: if comp_only: if cube.coord(coord).standard_name != \ coord_dict[coord]['standard_name']: inconsistency_sn.add(coord) if cube.coord(coord).long_name != \ coord_dict[coord]['long_name']: inconsistency_ln.add(coord) if cube.coord(coord).var_name != \ coord_dict[coord]['var_name']: inconsistency_vn.add(coord) if cube.coord(coord).attributes != \ coord_dict[coord]['attributes']: inconsistency_attr.add(coord) else: try: cube.coord(coord).standard_name = \ coord_dict[coord]['standard_name'] cube.coord(coord).long_name = \ coord_dict[coord]['long_name'] cube.coord(coord).var_name = \ coord_dict[coord]['var_name'] cube.coord(coord).attributes = \ coord_dict[coord]['attributes'] except ValueError: pass if any([inconsistency_sn, inconsistency_ln, inconsistency_vn, inconsistency_attr]): logger.info("\ncube dim coordinates differ: \n") if comp_only: log_inconsistent(list(inconsistency_sn), 'coords standard_name') log_inconsistent(list(inconsistency_ln), 'coords long_name') log_inconsistent(list(inconsistency_vn), 'coords var_name') log_inconsistent(list(inconsistency_attr), 'coords attributes') return cubes
def __init__(self, cube): self.cube = cube self.logger = log_module() self.message = ''