def _open_input(self): self.in_headers = next(self.reader, []) if len(self.in_headers) <= self.max_pos: raise DataException( 'Data in has %d columns, too little for max position %d', len(self.in_headers), self.max_pos) self.log.debug("csv has %d columns", len(self.in_headers))
def _value(self, v): if isstr(v): if v == "": return None try: v = p.parse(v) except ValueError: raise DataException("%s could not parse date %s", self.name, v) return v
def emit(self, v, escaper=None): """Emit is overwritten to add the to_json option""" if v is None: v = self.default if self.default is not None else v if self.strict: try: v = self.data_map[v] except KeyError: raise DataException("%s - %s not in map" % (self.name, v)) else: v = self.data_map.get( v, self.default if self.default is not None else v) if self.as_json: v = json.dumps(v) return super(MappingField, self).emit(v, escaper)
def update(self, **kwargs): '''Update method for chaining operations. Returns: self, so that methods can be chained Raises: :exc:`~.DataException`: raised if trying to set non defined field and strict model. ''' _meta = self.__class__._meta _fields = self.__class__._meta.fields.keys() for k, v in kwargs.items(): if k in _fields or not _meta.strict: setattr(self, k, v) else: raise DataException("trying to set unknown field %s" % k) return self
def __init__(self, **kwargs): _meta = self.__class__._meta # set value fields from kwargs if declared # if strict raise those not declared _fields = _meta.fields f = list(_fields.keys())[:] for k, v in kwargs.items(): if k == _meta.remark: setattr(self, k, v) elif k in f: setattr(self, k, _fields[k]._value(v)) f.remove(k) elif _meta.strict: raise DataException("trying to set unknown field %s" % k) else: setattr(self, k, v) # add missing fields for k in f: _f = _fields[k] setattr(self, k, _f._value(_f.default))
def scan(self, row): '''scan row and harvest distinct value. Takes a row of data and parses the required fields out of this. Args: row (list): array of source data Returns: parsed and processed value. Raises: :class:`~.ValidationException`: raised if explicit validation fails. ''' # see if we want to read a column in the row v = None if self.pos >= 0: try: _v = row[self.pos] except: raise DataException( 'parsing %r, row len %d, index %d not found', self.name, len(row), self.pos) # do null check if enabled if self.nullable is not None and _v == self.nullable: return v v = _v if self.validate and not self.validate(v): raise ValidationException( 'field %r input data did not validate' % self.name) # apply intermediate function on output, default is stripping if self.parse: v = self.parse(v) elif self.parse: v = self.parse(row) or v # delegate to inner function, to reuse this logic return self._value(v)