def validate_all(self, value):
if len(value) != 2:
msg = 'There must be two elements in the sequence.'
raise DefinitionValidationError(self, msg)
if value[0] > value[1]:
msg = 'The second element must be >= the first element.'
raise DefinitionValidationError(self, msg)
def parse(self, value):
if value is None:
ret = None
else:
try:
itr = iter(value)
except TypeError:
itr = iter([value])
ret = [OcgParameter.parse(self, element) for element in itr]
if self.unique:
if len(set(ret)) < len(value):
raise (DefinitionValidationError(
self, 'Argument sequence must have unique elements.'))
for idx in range(len(ret)):
try:
ret[idx] = self.element_type(ret[idx])
except TypeError:
if type(ret[idx]) not in self.element_type:
raise (DefinitionValidationError(
self,
'Element type incorrect. Acceptable types are: {0}'
.format(self.element_type)))
else:
pass
except ValueError:
raise (DefinitionValidationError(
self,
'Element type incorrect. Acceptable types are: {0}'.
format(self.element_type)))
ret = self.parse_all(ret)
self.validate_all(ret)
return (ret)
def _set_value_(self, value):
input_types = self.input_types + [basestring, NoneType]
type_matches = map(lambda x: isinstance(value, x), input_types)
if not any(type_matches):
raise (DefinitionValidationError(
self,
'Input value type "{1}" is not in accepted types: {0}'.format(
input_types, type(value))))
if isinstance(value, basestring):
value = self.parse_string(value)
else:
value = deepcopy(value)
ret = self.parse(value)
try:
if ret is not None:
try:
if self.return_type != type(ret):
ret = self.return_type(ret)
except:
if type(ret) not in self.return_type:
raise
except:
raise (DefinitionValidationError(self,
'Return type does not match.'))
self.validate(ret)
self._value = ret
def _validate_(self, value):
# The 'all' parameter will be reduced to a string eventually
if len(value) == 1 and value[0] == 'all':
pass
else:
try:
for val in value:
if val not in self._standard_groups:
raise DefinitionValidationError(
self, 'not in standard groups: {}'.format(
self._standard_groups))
# The grouping may not be a date part but a seasonal aggregation
except DefinitionValidationError:
months = list(range(1, 13))
for element in value:
# The keyword year and unique are okay for seasonal aggregations
if element in self._flags:
continue
elif isinstance(element, six.string_types):
if element not in self._flags:
raise DefinitionValidationError(
self,
'element not in flags: {}'.format(self._flags))
else:
for month in element:
if month not in months:
raise DefinitionValidationError(
self,
'month not in months: {}'.format(months))
def validate_all(self,values):
if len(values) == 0:
msg = 'At least one header value must be passed.'
raise(DefinitionValidationError(self,msg))
if not self.valid.issuperset(values):
msg = 'Valid headers are {0}.'.format(list(self.valid))
raise(DefinitionValidationError(self,msg))
def validate_definition(cls, definition):
AbstractFunction.validate_definition(definition)
assert isinstance(definition, dict)
from ocgis.ops.parms.definition import Calc
key = constants.CALC_KEY_KEYWORDS
if key not in definition:
msg = 'Keyword arguments are required using the "{0}" key: {1}'.format(
key, cls.parms_definition)
raise DefinitionValidationError(Calc, msg)
else:
kwds = definition[key]
try:
required = cls.required_variables
except AttributeError:
# this function likely does not have required variables and is not a multivariate function
assert not issubclass(cls, AbstractMultivariateFunction)
else:
kwds = kwds.copy()
for r in required:
kwds.pop(r, None)
if not set(kwds.keys()).issubset(list(
cls.parms_definition.keys())):
msg = 'Keyword arguments incorrect. Correct keyword arguments are: {0}'.format(
cls.parms_definition)
raise DefinitionValidationError(Calc, msg)
if cls.parms_required is not None:
for k in cls.parms_required:
if k not in kwds:
msg = 'The keyword parameter "{0}" is required.'.format(k)
raise DefinitionValidationError(Calc, msg)
def _validate_(self, value):
if len(value) == 0:
msg = 'Empty dictionaries are not allowed for optimizations. Use None instead.'
raise DefinitionValidationError(self, msg)
if set(value.keys()).issubset(set(self._allowed_keys)) == False:
msg = 'Allowed optimization keys are "{0}".'.format(
self._allowed_keys)
raise DefinitionValidationError(self, msg)
def validate_ops(cls, ops):
from ocgis.ops.parms.definition import OutputFormat
if len(list(ops.dataset)) > 1:
msg = 'Only one request dataset allowed for "{0}".'.format(
constants.OutputFormatName.METADATA_JSON)
raise DefinitionValidationError(OutputFormat, msg)
else:
for element in ops.dataset:
if isinstance(element, Field):
msg = 'Fields may not be converted to "{0}".'.format(
constants.OutputFormatName.METADATA_JSON)
raise DefinitionValidationError(OutputFormat, msg)
def parse_string(self, value):
if isinstance(value, six.string_types):
if os.path.isdir(value):
exc = DefinitionValidationError(
self, 'The provided path is a directory.')
ocgis_lh(exc=exc, logger='definition')
elements = value.split('|')
try:
elements = [float(e) for e in elements]
# switch geometry creation based on length. length of 2 is a point otherwise a bounding box
if len(elements) == 2:
geom = Point(elements[0], elements[1])
else:
minx, miny, maxx, maxy = elements
geom = Polygon(
((minx, miny), (minx, maxy), (maxx, maxy), (maxx, miny)))
if not geom.is_valid:
raise DefinitionValidationError(
self, 'Parsed geometry is not valid.')
ret = [{'geom': geom, 'properties': {'ugid': 1}}]
self._bounds = elements
# try the value as a key or path
except ValueError:
# if the path exists, then assume it is a path to a shapefile, otherwise assume it is a key
kwds = {}
if os.path.exists(value):
kwds['path'] = value
else:
kwds['key'] = value
# this is saved for later use by the openclimategis metadata output as the input value is inherently
# transformed
self._shp_key = value
# get the select_ugid test value
try:
test_value = self.select_ugid.value
# it may not have been passed as a parameter object
except AttributeError:
test_value = self.select_ugid
if test_value is None:
select_ugid = None
else:
select_ugid = test_value
kwds['select_uid'] = select_ugid
kwds['select_sql_where'] = self.geom_select_sql_where
kwds['uid'] = self.geom_uid
kwds[KeywordArgument.UNION] = self.union
kwds[KeywordArgument.DATA_MODEL] = self.data_model
ret = GeomCabinetIterator(**kwds)
return ret
def parse(self, value, check_basestrings=True):
if value is None:
ret = None
else:
try:
itr = iter(value)
except TypeError:
itr = iter([value])
ret = [AbstractParameter.parse(self, element) for element in itr]
if self.unique:
try:
if len(set(ret)) < len(value):
raise DefinitionValidationError(
self, 'Element sequences must be unique.')
# elements may not be reduceable to a set. attempt to reduce the individual elements and compare by
# this method.
except TypeError:
for start_idx, element in enumerate(value):
# some element sequences may have string flags which should be ignored.
if not check_basestrings:
if isinstance(element, six.string_types):
continue
element_set = set(element)
# Individual element sequences must be unique (i.e. [1,1,2] is not acceptable)
if len(element_set) != len(element):
raise DefinitionValidationError(
self, 'Element sequences must be unique.')
# this compares the uniqueness of an element in the sequence to other components in the
# sequence.
for to_check in range(start_idx + 1, len(value)):
if len(
element_set.intersection(
set(value[to_check]))) > 0:
raise DefinitionValidationError(self, '?')
for idx in range(len(ret)):
msg_exc = 'Element type incorrect. Acceptable types are: {0}'
try:
ret[idx] = self.element_type(ret[idx])
except TypeError:
if type(ret[idx]) not in self.element_type:
raise DefinitionValidationError(
self, msg_exc.format(self.element_type))
else:
pass
except ValueError:
raise DefinitionValidationError(
self, msg_exc.format(self.element_type))
ret = self.parse_all(ret)
self.validate_all(ret)
return ret
def _format_time_region_(self):
if isinstance(self.time_region, basestring):
ret = {}
parts = self.time_region.split('|')
for part in parts:
tpart, values = part.split('~')
try:
values = map(int, values.split('-'))
## may be nonetype
except ValueError:
if isinstance(values, basestring):
if values.lower() == 'none':
values = None
else:
raise
if values is not None and len(values) > 1:
values = range(values[0], values[1] + 1)
ret.update({tpart: values})
else:
ret = self.time_region
## add missing keys
for add_key in ['month', 'year']:
if add_key not in ret:
ret.update({add_key: None})
## confirm only month and year keys are present
for key in ret.keys():
if key not in ['month', 'year']:
raise (DefinitionValidationError(
'dataset', 'time regions keys must be month and/or year'))
self.time_region = ret
def validate(cls, ops):
if ops.calc_grouping is None:
from ocgis.ops.parms.definition import Calc
msg = 'Set functions must have a temporal grouping.'
ocgis_lh(exc=DefinitionValidationError(Calc, msg),
logger='calc.base')
def _parse_string_(self, value):
try:
key, uname = value.split('~', 1)
try:
uname, kwds_raw = uname.split('!', 1)
kwds_raw = kwds_raw.split('!')
kwds = OrderedDict()
for kwd in kwds_raw:
kwd_name, kwd_value = kwd.split('~')
try:
kwds.update({kwd_name: float(kwd_value)})
except ValueError:
kwds.update({kwd_name: str(kwd_value)})
except ValueError:
kwds = OrderedDict()
ret = {'func': key, 'name': uname, 'kwds': kwds}
except ValueError:
# likely a string to use for an eval function
if '=' not in value:
msg = 'String may not be parsed: "{0}".'.format(value)
raise DefinitionValidationError(self, msg)
else:
self._is_eval_function = True
ret = value
return ret
def parse_string(self,value):
elements = value.split('|')
try:
elements = [float(e) for e in elements]
## switch geometry creation based on length. length of 2 is a point
## otherwise a bounding box
if len(elements) == 2:
geom = Point(elements[0],elements[1])
else:
minx,miny,maxx,maxy = elements
geom = Polygon(((minx,miny),
(minx,maxy),
(maxx,maxy),
(maxx,miny)))
if not geom.is_valid:
raise(DefinitionValidationError(self,'Parsed geometry is not valid.'))
ret = GeometryDataset(1,geom)
self._bounds = elements
except ValueError:
self._shp_key = value
## get the select_ugid test value.
try:
test_value = self.select_ugid.value
except AttributeError:
test_value = self.select_ugid
if test_value is None:
select_ugid = None
else:
select_ugid = test_value
ret = ShpDataset(value,select_ugid=select_ugid)
return(ret)
def validate(self, value):
if value is not None:
try:
get_units_object(value)
except ValueError:
msg = 'Units not recognized by conversion backend: {}'.format(
value)
raise DefinitionValidationError(self.__class__, msg)
def _format_level_range_(self):
try:
ret = [int(v) for v in self.level_range.split('|')]
except AttributeError:
ret = self.level_range
if ret[0] > ret[1]:
raise(DefinitionValidationError('dataset','Level ordination incorrect.'))
self.level_range = ret
def validate(cls, ops):
if ops.calc_sample_size:
from ocgis.ops.parms.definition import CalcSampleSize
exc = DefinitionValidationError(
CalcSampleSize,
'Multivariate functions do not calculate sample size at this time.'
)
ocgis_lh(exc=exc, logger='calc.base')
# ensure the required variables are present
should_raise = False
for c in ops.calc:
if c['func'] == cls.key:
kwds = c['kwds']
# Check the required variables are keyword arguments.
if not len(
set(kwds.keys()).intersection(
set(cls.required_variables))) >= 2:
should_raise = True
break
# Ensure the mapped aliases exist.
fnames = []
for d in ops.dataset:
try:
for r in get_iter(d.rename_variable):
fnames.append(r)
except AttributeError:
# Fields do not have a rename variable attribute.
fnames += list(d.keys())
for xx in cls.required_variables:
to_check = kwds[xx]
if to_check not in fnames:
should_raise = True
break
if should_raise:
from ocgis.ops.parms.definition import Calc
msg = 'These field names are missing for multivariate function "{0}": {1}.'
exc = DefinitionValidationError(
Calc, msg.format(cls.__name__, cls.required_variables))
ocgis_lh(exc=exc, logger='calc.base')
def _validate_(self, value):
if not self._is_eval_function:
# get the aliases of the calculations
aliases = [ii['name'] for ii in value]
if len(aliases) != len(set(aliases)):
raise DefinitionValidationError(
self,
'User-provided calculation aliases must be unique: {0}'.
format(aliases))
for v in value:
if set(v.keys()) != self._required_keys_final:
msg = 'Required keys are: {0}'.format(
self._required_keys_final)
raise DefinitionValidationError(self, msg)
# run class-level definition
v[constants.CALC_KEY_CLASS_REFERENCE].validate_definition(v)
def validate(self,value):
''':rtype: void'''
if value is None:
if not self.nullable:
raise(DefinitionValidationError(self,'Argument is not nullable.'))
else:
pass
else:
self._validate_(value)
def _format_time_range_(self):
try:
ret = [datetime.strptime(v,'%Y-%m-%d') for v in self.time_range.split('|')]
ref = ret[1]
ret[1] = datetime(ref.year,ref.month,ref.day,23,59,59)
except AttributeError:
ret = self.time_range
if ret[0] > ret[1]:
raise(DefinitionValidationError('dataset','Time ordination incorrect.'))
self.time_range = ret
def _parse_(self,value):
if value is None:
ret = slice(None)
elif type(value) == int:
ret = slice(value,value+1)
elif type(value) in [list,tuple]:
ret = slice(*value)
else:
raise(DefinitionValidationError(self,'"{0}" cannot be converted to a slice object'.format(value)))
return(ret)
def validate(self, value):
"""
:raises: DefinitionValidationError
"""
if value is None:
if not self.nullable:
raise DefinitionValidationError(self,
'Argument is not nullable.')
else:
self._validate_(value)
def _format_(self):
if self.time_range is not None:
self._format_time_range_()
if self.time_region is not None:
self._format_time_region_()
if self.level_range is not None:
self._format_level_range_()
## ensure the time range and region overlaps
if not validate_time_subset(self.time_range, self.time_region):
raise (DefinitionValidationError(
'dataset', 'time_range and time_region do not overlap'))
def validate_ops(cls, ops):
"""
:param ops: An operation object to validate.
:type ops: :class:`~ocgis.OcgOperations`
:raises: DefinitionValidationError
"""
if cls.output_formats != 'all':
if ops.output_format not in cls.output_formats:
msg = 'Output format not supported for driver "{0}". Supported output formats are: {1}'.format(cls.key,
cls.output_formats)
ocgis_lh(logger='driver', exc=DefinitionValidationError('output_format', msg))
def parse(self,value):
if type(value) in [Polygon,MultiPolygon,Point]:
ret = GeometryDataset(1,value)
elif type(value) in [list,tuple]:
if len(value) in (2,4):
ret = self.parse_string('|'.join(map(str,value)))
else:
raise(DefinitionValidationError(self,'Bounding coordinates passed with length not equal to 2 (point) or 4 (bounding box).'))
elif isinstance(value,ShpDataset):
self._shp_key = value.key
ret = value
else:
ret = value
return(ret)
def _parse_(self, value):
for key in list(value.keys()):
if key not in list(self.default.keys()):
msg = 'The option "{0}" is not allowed.'.format(key)
raise DefinitionValidationError(self, msg)
if 'regrid_method' not in value:
value['regrid_method'] = 'auto'
if 'value_mask' not in value:
value['value_mask'] = None
if not any(
isinstance(value['value_mask'], t)
for t in self._possible_value_mask_types):
msg = 'Types for "value_mask" are: {0}'.format(
self._possible_value_mask_types)
raise DefinitionValidationError(self, msg)
if isinstance(value['value_mask'], np.ndarray):
if value['value_mask'].dtype != np.bool:
msg = '"value_mask" must be a boolean array.'
raise DefinitionValidationError(self, msg)
return value
def _parse_(self, value):
if value is not None:
# Add missing keys
for add_key in ['month', 'year']:
if add_key not in value:
value.update({add_key: None})
# Confirm only month and year keys are present
for key in list(value.keys()):
if key not in ['month', 'year']:
raise DefinitionValidationError(
self, 'Only "month" and "year" keys allowed.')
if all([i is None for i in list(value.values())]):
value = None
return value
def _set_value_(self, value):
if self._check_input_type:
input_types = self.input_types + list(six.string_types)
type_matches = [isinstance(value, x) for x in input_types]
if not any(type_matches) and value is not None:
msg = 'Input value type "{1}" is not in accepted types: {0}'
raise DefinitionValidationError(
self, msg.format(input_types, type(value)))
if isinstance(value, six.string_types):
value = self.parse_string(value)
else:
if self._perform_deepcopy:
value = deepcopy(value)
ret = self.parse(value)
if self._check_output_type:
try:
if ret is not None:
try:
if self.return_type != type(ret):
ret = self.return_type(ret)
except:
if type(ret) not in self.return_type:
if not any([
isinstance(ret, rt)
for rt in self.return_type
]):
raise
except:
raise DefinitionValidationError(self,
'Return type does not match.')
self.validate(ret)
self._value = ret
# Final hook for any modifications to the object.
self.finalize()
def _parse_(self, value):
# Get the request dataset from the collection if the value is a string.
if isinstance(value, six.string_types):
value_to_find = value
value = None
for d in list(self.dataset):
if d.field_name == value_to_find:
value = d
if value is None:
msg = 'String regrid destination "{}" not found.'.format(
value_to_find)
raise DefinitionValidationError(self.__class__, msg)
elif value is None:
is_regrid_destination = [
rd.regrid_destination for rd in list(self.dataset)
]
if sum(is_regrid_destination) > 1:
msg = 'Only one request dataset may be the destination regrid dataset.'
raise DefinitionValidationError(self, msg)
elif any(is_regrid_destination):
for rd in self.dataset:
if rd.regrid_destination:
value = rd
break
# If there is a destination, ensure at least one dataset is a source.
if value is not None:
if sum([d.regrid_source for d in list(self.dataset)]) < 1:
msg = 'There is a destination reqrid target, but no datasets are set as a source.'
raise DefinitionValidationError(self, msg)
# Return a field if the value is a request dataset.
if isinstance(value, RequestDataset):
value = value.get()
return value
def reduce_query(query):
## parse keys into groups.
groups = {}
ungrouped = {}
exp_key = re.compile('^(\D+)\d+$')
exp_number = re.compile('^\D+(\d+)$')
for key, value in query.iteritems():
try:
m_key = re.match(exp_key, key).groups()[0]
m_number = int(re.match(exp_number, key).groups()[0])
if m_key not in groups:
groups[m_key] = []
groups[m_key].append(m_number)
except AttributeError:
ungrouped[key] = value
## sort the groups
for value in groups.itervalues():
value.sort()
## ensure the groups are the same. only applicable if there are any
## grouped variables.
if len(groups) > 0:
arch = groups['uri']
for key, value in groups.iteritems():
try:
assert (arch == value)
except AssertionError:
if key in ['uri', 'variable']:
raise (DefinitionValidationError(
'reduce_query',
'Integer group indicators are not consistent.'))
else:
fill = [None] * len(arch)
for integer in value:
idx = arch.index(integer)
fill[idx] = integer
groups[key] = fill
## replace integers with actual values
for key, value in groups.iteritems():
for idx in range(len(value)):
if value[idx] is None:
continue
pull_key = key + str(value[idx])
value[idx] = query[pull_key][0]
groups[key] = [value]
## merge the grouped and ungrouped parameters
groups.update(ungrouped)
return (groups)
