def _is_insert(self):
"""
Returns true if the coverage should be inserted, false if only updates are needed
:rtype: bool
"""
cov = CoverageUtil(self.coverage.coverage_id)
return not cov.exists()
def update_lon_axis_to_epsg_version_85_if_needed(coverage_id,
crs_axes,
axes_configurations=None):
"""
NOTE: since rasdaman version 9.7+, in SECORE def/crs/EPSG/0 points to the newest EPSG version (e.g: 9.4.2 instead
of 8.5 as before). The problem is for EPSG:4326, Longitude axis's abbreviation changes from "Long" -> "Lon".
This method is used to allow import slices to existing coverage ("Lat Long" axes).
:param str coverage_id: existing coverage
:param list[CRSAxis] crs_axes: parsed CRSAxes from SECORE URL (e.g: def/crs/EPSG/4326 returns 2 CRSAxes: Lat, Lon)
:param dict{ axis1(dict), axis2(dict),... }: dictionary of configurations from ingredient file
"""
cov = CoverageUtil(coverage_id)
# Case 1: Coverage exists with "Lat Long" axes
if len(CRSUtil.coverage_axis_labels) == 0:
if cov.exists():
CRSUtil.coverage_axis_labels = cov.get_axes_labels()
for index, axis_label in enumerate(
CRSUtil.coverage_axis_labels):
if axis_label == CRSUtil.LONG_AXIS_LABEL_EPSG_VERSION_85:
CRSUtil.log_crs_replacement_epsg_version_0_by_version_85(
)
crs_axes[
index].label = CRSUtil.LONG_AXIS_LABEL_EPSG_VERSION_85
break
# Case 2: Coverage not exist, but in ingredient file for general recipes, it contains configuration for "Lat Long" axes
if axes_configurations is not None:
for key, value in axes_configurations.items():
CRSUtil.coverage_axis_labels.append(key)
for crs_axis in crs_axes:
# "Long" axis exists in configuration for ingredient file
if key == CRSUtil.LONG_AXIS_LABEL_EPSG_VERSION_85:
if crs_axis.label == CRSUtil.LONG_AXIS_LABEL_EPSG_VERSION_0:
crs_axis.label = CRSUtil.LONG_AXIS_LABEL_EPSG_VERSION_85
break
def describe(self):
"""
This methods is called before insert or update is run. You should override the method and add any comments
regarding the operations that you will perform via log.info to inform the user. You should explicitly state
the information that you deduced (e.g. timestamps for a timeseries) so that the consequences are clear.
"""
cov = CoverageUtil(self.session.get_coverage_id())
operation_type = "UPDATE" if cov.exists() else "INSERT"
log.info("The recipe has been validated and is ready to run.")
log.info(make_bold("Recipe: ") + self.session.get_recipe()['name'])
log.info(make_bold("Coverage: ") + self.session.get_coverage_id())
log.info(make_bold("WCS Service: ") + ConfigManager.wcs_service)
log.info(make_bold("Operation: ") + operation_type)
log.info(
make_bold("Subset Correction: ") +
str(ConfigManager.subset_correction))
log.info(make_bold("Mocked: ") + str(ConfigManager.mock))
log.info(make_bold("WMS Import: ") + str(self.session.wms_import))
# Blocking means analyzing all input files before importing all coverage slices
# Non-blocking means analazying 1 file then import 1 file then continue with next file.
import_mode = "Blocking"
if not self.session.blocking:
import_mode = "Non-blocking"
log.info(make_bold("Import mode: ") + import_mode)
if ConfigManager.track_files:
log.info(
make_bold("Track files: ") + str(ConfigManager.track_files))
if ConfigManager.skip:
log.info(make_bold("Skip: ") + str(ConfigManager.skip))
if ConfigManager.retry:
log.info(make_bold("Retries: ") + str(ConfigManager.retries))
if ConfigManager.slice_restriction is not None:
log.info(
make_bold("Slice Restriction: ") +
str(ConfigManager.slice_restriction))
pass
def describe(self):
"""
This methods is called before insert or update is run. You should override the method and add any comments
regarding the operations that you will perform via log.info to inform the user. You should explicitly state
the information that you deduced (e.g. timestamps for a timeseries) so that the consequences are clear.
"""
cov = CoverageUtil(self.session.get_coverage_id())
operation_type = "UPDATE" if cov.exists() else "INSERT"
log.info("The recipe has been validated and is ready to run.")
log.info("\033[1mRecipe:\x1b[0m " + self.session.get_recipe()['name'])
log.info("\033[1mCoverage:\x1b[0m " + self.session.get_coverage_id())
log.info("\033[1mWCS Service:\x1b[0m " + ConfigManager.wcs_service)
log.info("\033[1mOperation:\x1b[0m " + operation_type)
log.info("\033[1mSubset Correction:\x1b[0m " + str(ConfigManager.subset_correction))
log.info("\033[1mMocked:\x1b[0m " + str(ConfigManager.mock))
if ConfigManager.track_files:
log.info("\033[1mTrack files:\x1b[0m " + str(ConfigManager.track_files))
if ConfigManager.skip:
log.info("\033[1mSkip:\x1b[0m " + str(ConfigManager.skip))
if ConfigManager.retry:
log.info("\033[1mRetries:\x1b[0m " + str(ConfigManager.retries))
if ConfigManager.slice_restriction is not None:
log.info("\033[1mSlice Restriction:\x1b[0m " + str(ConfigManager.slice_restriction))
pass
def validate_base(self, ignore_no_files=False):
"""
Validates the configuration and the input files
:param bool ignore_no_files: if the extending recipe does not work with files, set this to true to skip
the validation check for no files (used in wcs_extract recipe).
"""
if self.session.get_wcs_service(
) is None or self.session.get_wcs_service() == "":
raise RecipeValidationException("No valid wcs endpoint provided")
if self.session.get_crs_resolver(
) is None or self.session.get_crs_resolver() == "":
raise RecipeValidationException("No valid crs resolver provided")
if self.session.get_coverage_id(
) is None or self.session.get_coverage_id() == "":
raise RecipeValidationException("No valid coverage id provided")
import recipes.virtual_coverage.recipe as super_coverage
if self.session.get_recipe_name() == super_coverage.Recipe.RECIPE_NAME:
# NOTE: virtual_coverage recipe does not require any input files
return
if not FileUtil.check_dir_writable(ConfigManager.tmp_directory):
raise RecipeValidationException(
"Cannot write to tmp directory '{}'".format(
ConfigManager.tmp_directory))
checked_files = []
for file in self.session.get_files():
if FileUtil.validate_file_path(file.get_filepath()):
checked_files.append(file)
if not ignore_no_files:
# If no input file is available, exit wcst_import.
FileUtil.validate_input_file_paths(checked_files)
self.session.files = checked_files
if 'wms_import' not in self.options:
self.options['wms_import'] = False
else:
self.options['wms_import'] = bool(self.options['wms_import'])
if 'tiling' not in self.options:
self.options['tiling'] = None
if 'scale_levels' not in self.options:
self.options['scale_levels'] = None
if 'scale_factors' not in self.options:
self.options['scale_factors'] = None
if self.options['scale_levels'] is not None \
and self.options['scale_factors'] is not None:
raise RecipeValidationException(
"Only one of 'scale_levels' or 'scale_factors' "
"setting can exist in the ingredients file.")
if self.options['scale_factors'] is not None:
# as scale_factors and scale_levels are only valid when initializing a new coverage
cov = CoverageUtil(self.session.get_coverage_id())
if not cov.exists():
for obj in self.options['scale_factors']:
if 'coverage_id' not in obj or 'factors' not in obj:
raise RecipeValidationException(
"All elements of 'scale_factors' list must contain "
"'coverage_id' and 'factors' properties")
coverage_id = obj['coverage_id']
cov = CoverageUtil(coverage_id)
if cov.exists():
raise RecipeValidationException(
"Downscaled level coverage '" + coverage_id +
"' already exists, "
"please use a different 'coverage_id' in 'scale_factors' list"
)
self.validate_pyramid_members()
self.validate_pyramid_bases()
if "import_order" in self.options:
if self.options['import_order'] != AbstractToCoverageConverter.IMPORT_ORDER_ASCENDING \
and self.options['import_order'] != AbstractToCoverageConverter.IMPORT_ORDER_DESCENDING:
error_message = "'import_order' option must be '{}' or '{}', given '{}'.".\
format(AbstractToCoverageConverter.IMPORT_ORDER_ASCENDING,
AbstractToCoverageConverter.IMPORT_ORDER_DESCENDING,
self.options['import_order'])
raise RecipeValidationException(error_message)
else:
self.options['import_order'] = None
def _adjust_irregular_axis_geo_lower_bound(self, coverage_id, axis_label,
axis_type, is_day_unit,
axis_geo_lower_bound,
slice_group_size):
"""
(!) Use only if irregular axis's dataBound is False, then adjust it's geo lower bound by current coverage's axis lower bound.
e.g: coverage's axis lower bound is '2015-01-10' and slice_group_size = 5 then irregular axis's lower bound
from '2015-01-08' or '2015-01-13' will be adjusted to '2015-01-10'.
:param int slice_group_size: a positive integer
"""
resolution = IrregularUserAxis.DEFAULT_RESOLUTION
if axis_label not in self.irregular_axis_geo_lower_bound_dict:
# Need to parse it from coverage's DescribeCoverage result
cov = CoverageUtil(coverage_id)
if cov.exists():
axes_labels = cov.get_axes_labels()
# Get current coverage's axis geo lower bound
i = axes_labels.index(axis_label)
lower_bounds = cov.get_axes_lower_bounds()
coverage_geo_lower_bound = lower_bounds[i]
# Translate datetime format to seconds
if axis_type == UserAxisType.DATE:
coverage_geo_lower_bound = arrow.get(
coverage_geo_lower_bound).float_timestamp
axis_geo_lower_bound = arrow.get(
axis_geo_lower_bound).float_timestamp
else:
# Coverage does not exist, first InsertCoverage request
self.irregular_axis_geo_lower_bound_dict[
axis_label] = axis_geo_lower_bound
return axis_geo_lower_bound
else:
coverage_geo_lower_bound = self.irregular_axis_geo_lower_bound_dict[
axis_label]
if is_day_unit:
# AnsiDate (8600 seconds / day)
slice_group_size = slice_group_size * DateTimeUtil.DAY_IN_SECONDS
# e.g: irregular axis's level with current coverage's lower bound is 5, axis's lower bound is 12
# and sliceGroupSize = 5. Result is: (12 - 5) / (1 * 5) = 1
distance = math.floor(
(axis_geo_lower_bound - coverage_geo_lower_bound) /
(resolution * slice_group_size))
# So the adjusted value is 5 + 1 * 1 * 5 = 10
adjusted_bound = coverage_geo_lower_bound + (distance * resolution *
slice_group_size)
if adjusted_bound < coverage_geo_lower_bound:
# Save this one to cache for further processing
self.irregular_axis_geo_lower_bound_dict[
axis_label] = adjusted_bound
return adjusted_bound