def _create_ancil_datasets(self, model_run, session): """ Create ancillary datasets if available and add them to the model run datasets :param model_run: Model run to update :param session: DB Session :return: """ soil_props_file = model_run.get_python_parameter_value(constants.JULES_PARAM_SOIL_PROPS_FILE) if soil_props_file is not None: dataset_type = self.get_dataset_type(constants.DATASET_TYPE_SOIL_PROP, session) self._create_dataset(dataset_type, soil_props_file, True, model_run, session) frac_file = model_run.get_python_parameter_value(constants.JULES_PARAM_FRAC_FILE) user_upload_id = session.query(DrivingDataset) \ .filter(DrivingDataset.name == constants.USER_UPLOAD_DRIVING_DATASET_NAME) \ .one().id is_user_uploaded = model_run.driving_dataset_id == user_upload_id if len(model_run.land_cover_actions) > 0 or (model_run.is_for_single_cell() and not is_user_uploaded): frac_file = 'run{model_id}/{user_edited_file}'.format( model_id=model_run.id, user_edited_file=constants.USER_EDITED_FRACTIONAL_FILENAME) if frac_file is not None and frac_file != constants.FRACTIONAL_FILENAME: dataset_type = self.get_dataset_type(constants.DATASET_TYPE_LAND_COVER_FRAC, session) frac_file_vis = insert_before_file_extension(frac_file, constants.MODIFIED_FOR_VISUALISATION_EXTENSION) self._create_dataset(dataset_type, frac_file_vis, True, model_run, session)
def generate_output_file_path(self, model_run_id, output_var_name, period, year, is_single_cell): """ Generate a file path for an output dataset :param model_run_id: ID of Model Run to download from :param output_var_name: Variable name to download :param period: Period (e.g. 'daily') :param year: Calendar year to download (only required if period is daily or hourly and the run is multicell) :param is_single_cell: Is the model run single cell? :return: File path (relative to run directory) """ name_template = "run{model_run_id}/{output_dir}/{run_id}.{var_name}_{period}.nc" file_name = name_template.format( model_run_id=model_run_id, output_dir=constants.OUTPUT_DIR, run_id=constants.RUN_ID, var_name=output_var_name, period=period.lower() ) if is_single_cell: year = None # Get all the data if year is not None and period.lower() not in ['monthly', 'yearly']: year_suffix = ".%s" % year file_name = insert_before_file_extension(file_name, year_suffix) return file_name
def get_file_locations(self, results): """ Return all the file locations referenced by the results parameters if there is an error add it to the error dictionary :param results: dictionary of results from the driving dataset page :return:list of locations """ locations = [] for key in ['land_frac_file', 'latlon_file', constants.LAND_COVER_FRAC_FILE_INPUT_NAME, 'soil_props_file']: self._check_location(key, results.get(key), self._errors, False) if constants.LAND_COVER_FRAC_FILE_INPUT_NAME in results: converted_filename = insert_before_file_extension( results.get(constants.LAND_COVER_FRAC_FILE_INPUT_NAME), constants.MODIFIED_FOR_VISUALISATION_EXTENSION) self._check_location(constants.LAND_COVER_FRAC_FILE_INPUT_NAME, converted_filename, self._errors, False) regions = results.get('region', []) region_errors = [] region_error = False for region, index in zip(regions, range(len(regions))): region_errors.append({}) local_error = not self._check_location('path', region['path'], region_errors[index], True) region_error = region_error or local_error if region_error: self._errors['region'] = region_errors driving_data_vars = results.get(constants.PREFIX_FOR_DRIVING_VARS, []) driving_data_errors = [] driving_data_error = False drive_file = results.get('drive_file') start_date = results.get('driving_data_start') end_date = results.get('driving_data_end') for driving_data, index in zip(driving_data_vars, range(len(driving_data_vars))): driving_data_errors.append({}) ncml_filename = self._get_ncml_filename(driving_data['templates'], drive_file) local_error = not self._check_location_and_add_to( 'templates', locations, ncml_filename, driving_data_errors[index], self._dataset_types[constants.DATASET_TYPE_COVERAGE], driving_data['vars']) driving_data_error = driving_data_error or local_error for filename in self._get_drive_filenames(driving_data['templates'], drive_file, start_date, end_date): local_error = not self._check_location( 'templates', filename, driving_data_errors[index], True) if local_error: driving_data_error = True break if driving_data_error: self._errors[constants.PREFIX_FOR_DRIVING_VARS] = driving_data_errors return locations
def test_insert_before_file_extension(self): path = "/home/user/data/file.name.here.nc" string_to_insert = "_INSERTED_STRING" modified_path = insert_before_file_extension(path, string_to_insert) expected_result = "/home/user/data/file.name.here_INSERTED_STRING.nc" assert_that(modified_path, is_(expected_result))