def validate_all(self, value): if len(value) != 2: msg = 'There must be two elements in the sequence.' raise DefinitionValidationError(self, msg) if value[0] > value[1]: msg = 'The second element must be >= the first element.' raise DefinitionValidationError(self, msg)
def parse(self, value): if value is None: ret = None else: try: itr = iter(value) except TypeError: itr = iter([value]) ret = [OcgParameter.parse(self, element) for element in itr] if self.unique: if len(set(ret)) < len(value): raise (DefinitionValidationError( self, 'Argument sequence must have unique elements.')) for idx in range(len(ret)): try: ret[idx] = self.element_type(ret[idx]) except TypeError: if type(ret[idx]) not in self.element_type: raise (DefinitionValidationError( self, 'Element type incorrect. Acceptable types are: {0}' .format(self.element_type))) else: pass except ValueError: raise (DefinitionValidationError( self, 'Element type incorrect. Acceptable types are: {0}'. format(self.element_type))) ret = self.parse_all(ret) self.validate_all(ret) return (ret)
def _set_value_(self, value): input_types = self.input_types + [basestring, NoneType] type_matches = map(lambda x: isinstance(value, x), input_types) if not any(type_matches): raise (DefinitionValidationError( self, 'Input value type "{1}" is not in accepted types: {0}'.format( input_types, type(value)))) if isinstance(value, basestring): value = self.parse_string(value) else: value = deepcopy(value) ret = self.parse(value) try: if ret is not None: try: if self.return_type != type(ret): ret = self.return_type(ret) except: if type(ret) not in self.return_type: raise except: raise (DefinitionValidationError(self, 'Return type does not match.')) self.validate(ret) self._value = ret
def _validate_(self, value): # The 'all' parameter will be reduced to a string eventually if len(value) == 1 and value[0] == 'all': pass else: try: for val in value: if val not in self._standard_groups: raise DefinitionValidationError( self, 'not in standard groups: {}'.format( self._standard_groups)) # The grouping may not be a date part but a seasonal aggregation except DefinitionValidationError: months = list(range(1, 13)) for element in value: # The keyword year and unique are okay for seasonal aggregations if element in self._flags: continue elif isinstance(element, six.string_types): if element not in self._flags: raise DefinitionValidationError( self, 'element not in flags: {}'.format(self._flags)) else: for month in element: if month not in months: raise DefinitionValidationError( self, 'month not in months: {}'.format(months))
def validate_all(self,values): if len(values) == 0: msg = 'At least one header value must be passed.' raise(DefinitionValidationError(self,msg)) if not self.valid.issuperset(values): msg = 'Valid headers are {0}.'.format(list(self.valid)) raise(DefinitionValidationError(self,msg))
def validate_definition(cls, definition): AbstractFunction.validate_definition(definition) assert isinstance(definition, dict) from ocgis.ops.parms.definition import Calc key = constants.CALC_KEY_KEYWORDS if key not in definition: msg = 'Keyword arguments are required using the "{0}" key: {1}'.format( key, cls.parms_definition) raise DefinitionValidationError(Calc, msg) else: kwds = definition[key] try: required = cls.required_variables except AttributeError: # this function likely does not have required variables and is not a multivariate function assert not issubclass(cls, AbstractMultivariateFunction) else: kwds = kwds.copy() for r in required: kwds.pop(r, None) if not set(kwds.keys()).issubset(list( cls.parms_definition.keys())): msg = 'Keyword arguments incorrect. Correct keyword arguments are: {0}'.format( cls.parms_definition) raise DefinitionValidationError(Calc, msg) if cls.parms_required is not None: for k in cls.parms_required: if k not in kwds: msg = 'The keyword parameter "{0}" is required.'.format(k) raise DefinitionValidationError(Calc, msg)
def _validate_(self, value): if len(value) == 0: msg = 'Empty dictionaries are not allowed for optimizations. Use None instead.' raise DefinitionValidationError(self, msg) if set(value.keys()).issubset(set(self._allowed_keys)) == False: msg = 'Allowed optimization keys are "{0}".'.format( self._allowed_keys) raise DefinitionValidationError(self, msg)
def validate_ops(cls, ops): from ocgis.ops.parms.definition import OutputFormat if len(list(ops.dataset)) > 1: msg = 'Only one request dataset allowed for "{0}".'.format( constants.OutputFormatName.METADATA_JSON) raise DefinitionValidationError(OutputFormat, msg) else: for element in ops.dataset: if isinstance(element, Field): msg = 'Fields may not be converted to "{0}".'.format( constants.OutputFormatName.METADATA_JSON) raise DefinitionValidationError(OutputFormat, msg)
def parse_string(self, value): if isinstance(value, six.string_types): if os.path.isdir(value): exc = DefinitionValidationError( self, 'The provided path is a directory.') ocgis_lh(exc=exc, logger='definition') elements = value.split('|') try: elements = [float(e) for e in elements] # switch geometry creation based on length. length of 2 is a point otherwise a bounding box if len(elements) == 2: geom = Point(elements[0], elements[1]) else: minx, miny, maxx, maxy = elements geom = Polygon( ((minx, miny), (minx, maxy), (maxx, maxy), (maxx, miny))) if not geom.is_valid: raise DefinitionValidationError( self, 'Parsed geometry is not valid.') ret = [{'geom': geom, 'properties': {'ugid': 1}}] self._bounds = elements # try the value as a key or path except ValueError: # if the path exists, then assume it is a path to a shapefile, otherwise assume it is a key kwds = {} if os.path.exists(value): kwds['path'] = value else: kwds['key'] = value # this is saved for later use by the openclimategis metadata output as the input value is inherently # transformed self._shp_key = value # get the select_ugid test value try: test_value = self.select_ugid.value # it may not have been passed as a parameter object except AttributeError: test_value = self.select_ugid if test_value is None: select_ugid = None else: select_ugid = test_value kwds['select_uid'] = select_ugid kwds['select_sql_where'] = self.geom_select_sql_where kwds['uid'] = self.geom_uid kwds[KeywordArgument.UNION] = self.union kwds[KeywordArgument.DATA_MODEL] = self.data_model ret = GeomCabinetIterator(**kwds) return ret
def parse(self, value, check_basestrings=True): if value is None: ret = None else: try: itr = iter(value) except TypeError: itr = iter([value]) ret = [AbstractParameter.parse(self, element) for element in itr] if self.unique: try: if len(set(ret)) < len(value): raise DefinitionValidationError( self, 'Element sequences must be unique.') # elements may not be reduceable to a set. attempt to reduce the individual elements and compare by # this method. except TypeError: for start_idx, element in enumerate(value): # some element sequences may have string flags which should be ignored. if not check_basestrings: if isinstance(element, six.string_types): continue element_set = set(element) # Individual element sequences must be unique (i.e. [1,1,2] is not acceptable) if len(element_set) != len(element): raise DefinitionValidationError( self, 'Element sequences must be unique.') # this compares the uniqueness of an element in the sequence to other components in the # sequence. for to_check in range(start_idx + 1, len(value)): if len( element_set.intersection( set(value[to_check]))) > 0: raise DefinitionValidationError(self, '?') for idx in range(len(ret)): msg_exc = 'Element type incorrect. Acceptable types are: {0}' try: ret[idx] = self.element_type(ret[idx]) except TypeError: if type(ret[idx]) not in self.element_type: raise DefinitionValidationError( self, msg_exc.format(self.element_type)) else: pass except ValueError: raise DefinitionValidationError( self, msg_exc.format(self.element_type)) ret = self.parse_all(ret) self.validate_all(ret) return ret
def _format_time_region_(self): if isinstance(self.time_region, basestring): ret = {} parts = self.time_region.split('|') for part in parts: tpart, values = part.split('~') try: values = map(int, values.split('-')) ## may be nonetype except ValueError: if isinstance(values, basestring): if values.lower() == 'none': values = None else: raise if values is not None and len(values) > 1: values = range(values[0], values[1] + 1) ret.update({tpart: values}) else: ret = self.time_region ## add missing keys for add_key in ['month', 'year']: if add_key not in ret: ret.update({add_key: None}) ## confirm only month and year keys are present for key in ret.keys(): if key not in ['month', 'year']: raise (DefinitionValidationError( 'dataset', 'time regions keys must be month and/or year')) self.time_region = ret
def validate(cls, ops): if ops.calc_grouping is None: from ocgis.ops.parms.definition import Calc msg = 'Set functions must have a temporal grouping.' ocgis_lh(exc=DefinitionValidationError(Calc, msg), logger='calc.base')
def _parse_string_(self, value): try: key, uname = value.split('~', 1) try: uname, kwds_raw = uname.split('!', 1) kwds_raw = kwds_raw.split('!') kwds = OrderedDict() for kwd in kwds_raw: kwd_name, kwd_value = kwd.split('~') try: kwds.update({kwd_name: float(kwd_value)}) except ValueError: kwds.update({kwd_name: str(kwd_value)}) except ValueError: kwds = OrderedDict() ret = {'func': key, 'name': uname, 'kwds': kwds} except ValueError: # likely a string to use for an eval function if '=' not in value: msg = 'String may not be parsed: "{0}".'.format(value) raise DefinitionValidationError(self, msg) else: self._is_eval_function = True ret = value return ret
def parse_string(self,value): elements = value.split('|') try: elements = [float(e) for e in elements] ## switch geometry creation based on length. length of 2 is a point ## otherwise a bounding box if len(elements) == 2: geom = Point(elements[0],elements[1]) else: minx,miny,maxx,maxy = elements geom = Polygon(((minx,miny), (minx,maxy), (maxx,maxy), (maxx,miny))) if not geom.is_valid: raise(DefinitionValidationError(self,'Parsed geometry is not valid.')) ret = GeometryDataset(1,geom) self._bounds = elements except ValueError: self._shp_key = value ## get the select_ugid test value. try: test_value = self.select_ugid.value except AttributeError: test_value = self.select_ugid if test_value is None: select_ugid = None else: select_ugid = test_value ret = ShpDataset(value,select_ugid=select_ugid) return(ret)
def validate(self, value): if value is not None: try: get_units_object(value) except ValueError: msg = 'Units not recognized by conversion backend: {}'.format( value) raise DefinitionValidationError(self.__class__, msg)
def _format_level_range_(self): try: ret = [int(v) for v in self.level_range.split('|')] except AttributeError: ret = self.level_range if ret[0] > ret[1]: raise(DefinitionValidationError('dataset','Level ordination incorrect.')) self.level_range = ret
def validate(cls, ops): if ops.calc_sample_size: from ocgis.ops.parms.definition import CalcSampleSize exc = DefinitionValidationError( CalcSampleSize, 'Multivariate functions do not calculate sample size at this time.' ) ocgis_lh(exc=exc, logger='calc.base') # ensure the required variables are present should_raise = False for c in ops.calc: if c['func'] == cls.key: kwds = c['kwds'] # Check the required variables are keyword arguments. if not len( set(kwds.keys()).intersection( set(cls.required_variables))) >= 2: should_raise = True break # Ensure the mapped aliases exist. fnames = [] for d in ops.dataset: try: for r in get_iter(d.rename_variable): fnames.append(r) except AttributeError: # Fields do not have a rename variable attribute. fnames += list(d.keys()) for xx in cls.required_variables: to_check = kwds[xx] if to_check not in fnames: should_raise = True break if should_raise: from ocgis.ops.parms.definition import Calc msg = 'These field names are missing for multivariate function "{0}": {1}.' exc = DefinitionValidationError( Calc, msg.format(cls.__name__, cls.required_variables)) ocgis_lh(exc=exc, logger='calc.base')
def _validate_(self, value): if not self._is_eval_function: # get the aliases of the calculations aliases = [ii['name'] for ii in value] if len(aliases) != len(set(aliases)): raise DefinitionValidationError( self, 'User-provided calculation aliases must be unique: {0}'. format(aliases)) for v in value: if set(v.keys()) != self._required_keys_final: msg = 'Required keys are: {0}'.format( self._required_keys_final) raise DefinitionValidationError(self, msg) # run class-level definition v[constants.CALC_KEY_CLASS_REFERENCE].validate_definition(v)
def validate(self,value): ''':rtype: void''' if value is None: if not self.nullable: raise(DefinitionValidationError(self,'Argument is not nullable.')) else: pass else: self._validate_(value)
def _format_time_range_(self): try: ret = [datetime.strptime(v,'%Y-%m-%d') for v in self.time_range.split('|')] ref = ret[1] ret[1] = datetime(ref.year,ref.month,ref.day,23,59,59) except AttributeError: ret = self.time_range if ret[0] > ret[1]: raise(DefinitionValidationError('dataset','Time ordination incorrect.')) self.time_range = ret
def _parse_(self,value): if value is None: ret = slice(None) elif type(value) == int: ret = slice(value,value+1) elif type(value) in [list,tuple]: ret = slice(*value) else: raise(DefinitionValidationError(self,'"{0}" cannot be converted to a slice object'.format(value))) return(ret)
def validate(self, value): """ :raises: DefinitionValidationError """ if value is None: if not self.nullable: raise DefinitionValidationError(self, 'Argument is not nullable.') else: self._validate_(value)
def _format_(self): if self.time_range is not None: self._format_time_range_() if self.time_region is not None: self._format_time_region_() if self.level_range is not None: self._format_level_range_() ## ensure the time range and region overlaps if not validate_time_subset(self.time_range, self.time_region): raise (DefinitionValidationError( 'dataset', 'time_range and time_region do not overlap'))
def validate_ops(cls, ops): """ :param ops: An operation object to validate. :type ops: :class:`~ocgis.OcgOperations` :raises: DefinitionValidationError """ if cls.output_formats != 'all': if ops.output_format not in cls.output_formats: msg = 'Output format not supported for driver "{0}". Supported output formats are: {1}'.format(cls.key, cls.output_formats) ocgis_lh(logger='driver', exc=DefinitionValidationError('output_format', msg))
def parse(self,value): if type(value) in [Polygon,MultiPolygon,Point]: ret = GeometryDataset(1,value) elif type(value) in [list,tuple]: if len(value) in (2,4): ret = self.parse_string('|'.join(map(str,value))) else: raise(DefinitionValidationError(self,'Bounding coordinates passed with length not equal to 2 (point) or 4 (bounding box).')) elif isinstance(value,ShpDataset): self._shp_key = value.key ret = value else: ret = value return(ret)
def _parse_(self, value): for key in list(value.keys()): if key not in list(self.default.keys()): msg = 'The option "{0}" is not allowed.'.format(key) raise DefinitionValidationError(self, msg) if 'regrid_method' not in value: value['regrid_method'] = 'auto' if 'value_mask' not in value: value['value_mask'] = None if not any( isinstance(value['value_mask'], t) for t in self._possible_value_mask_types): msg = 'Types for "value_mask" are: {0}'.format( self._possible_value_mask_types) raise DefinitionValidationError(self, msg) if isinstance(value['value_mask'], np.ndarray): if value['value_mask'].dtype != np.bool: msg = '"value_mask" must be a boolean array.' raise DefinitionValidationError(self, msg) return value
def _parse_(self, value): if value is not None: # Add missing keys for add_key in ['month', 'year']: if add_key not in value: value.update({add_key: None}) # Confirm only month and year keys are present for key in list(value.keys()): if key not in ['month', 'year']: raise DefinitionValidationError( self, 'Only "month" and "year" keys allowed.') if all([i is None for i in list(value.values())]): value = None return value
def _set_value_(self, value): if self._check_input_type: input_types = self.input_types + list(six.string_types) type_matches = [isinstance(value, x) for x in input_types] if not any(type_matches) and value is not None: msg = 'Input value type "{1}" is not in accepted types: {0}' raise DefinitionValidationError( self, msg.format(input_types, type(value))) if isinstance(value, six.string_types): value = self.parse_string(value) else: if self._perform_deepcopy: value = deepcopy(value) ret = self.parse(value) if self._check_output_type: try: if ret is not None: try: if self.return_type != type(ret): ret = self.return_type(ret) except: if type(ret) not in self.return_type: if not any([ isinstance(ret, rt) for rt in self.return_type ]): raise except: raise DefinitionValidationError(self, 'Return type does not match.') self.validate(ret) self._value = ret # Final hook for any modifications to the object. self.finalize()
def _parse_(self, value): # Get the request dataset from the collection if the value is a string. if isinstance(value, six.string_types): value_to_find = value value = None for d in list(self.dataset): if d.field_name == value_to_find: value = d if value is None: msg = 'String regrid destination "{}" not found.'.format( value_to_find) raise DefinitionValidationError(self.__class__, msg) elif value is None: is_regrid_destination = [ rd.regrid_destination for rd in list(self.dataset) ] if sum(is_regrid_destination) > 1: msg = 'Only one request dataset may be the destination regrid dataset.' raise DefinitionValidationError(self, msg) elif any(is_regrid_destination): for rd in self.dataset: if rd.regrid_destination: value = rd break # If there is a destination, ensure at least one dataset is a source. if value is not None: if sum([d.regrid_source for d in list(self.dataset)]) < 1: msg = 'There is a destination reqrid target, but no datasets are set as a source.' raise DefinitionValidationError(self, msg) # Return a field if the value is a request dataset. if isinstance(value, RequestDataset): value = value.get() return value
def reduce_query(query): ## parse keys into groups. groups = {} ungrouped = {} exp_key = re.compile('^(\D+)\d+$') exp_number = re.compile('^\D+(\d+)$') for key, value in query.iteritems(): try: m_key = re.match(exp_key, key).groups()[0] m_number = int(re.match(exp_number, key).groups()[0]) if m_key not in groups: groups[m_key] = [] groups[m_key].append(m_number) except AttributeError: ungrouped[key] = value ## sort the groups for value in groups.itervalues(): value.sort() ## ensure the groups are the same. only applicable if there are any ## grouped variables. if len(groups) > 0: arch = groups['uri'] for key, value in groups.iteritems(): try: assert (arch == value) except AssertionError: if key in ['uri', 'variable']: raise (DefinitionValidationError( 'reduce_query', 'Integer group indicators are not consistent.')) else: fill = [None] * len(arch) for integer in value: idx = arch.index(integer) fill[idx] = integer groups[key] = fill ## replace integers with actual values for key, value in groups.iteritems(): for idx in range(len(value)): if value[idx] is None: continue pull_key = key + str(value[idx]) value[idx] = query[pull_key][0] groups[key] = [value] ## merge the grouped and ungrouped parameters groups.update(ungrouped) return (groups)