def _open_input(self):
self.in_headers = next(self.reader, [])
if len(self.in_headers) <= self.max_pos:
raise DataException(
'Data in has %d columns, too little for max position %d',
len(self.in_headers), self.max_pos)
self.log.debug("csv has %d columns", len(self.in_headers))
def _value(self, v):
if isstr(v):
if v == "":
return None
try:
v = p.parse(v)
except ValueError:
raise DataException("%s could not parse date %s", self.name, v)
return v
def emit(self, v, escaper=None):
"""Emit is overwritten to add the to_json option"""
if v is None:
v = self.default if self.default is not None else v
if self.strict:
try:
v = self.data_map[v]
except KeyError:
raise DataException("%s - %s not in map" % (self.name, v))
else:
v = self.data_map.get(
v, self.default if self.default is not None else v)
if self.as_json:
v = json.dumps(v)
return super(MappingField, self).emit(v, escaper)
def update(self, **kwargs):
'''Update method for chaining operations.
Returns:
self, so that methods can be chained
Raises:
:exc:`~.DataException`: raised if trying to set non defined field
and strict model.
'''
_meta = self.__class__._meta
_fields = self.__class__._meta.fields.keys()
for k, v in kwargs.items():
if k in _fields or not _meta.strict:
setattr(self, k, v)
else:
raise DataException("trying to set unknown field %s" % k)
return self
def __init__(self, **kwargs):
_meta = self.__class__._meta
# set value fields from kwargs if declared
# if strict raise those not declared
_fields = _meta.fields
f = list(_fields.keys())[:]
for k, v in kwargs.items():
if k == _meta.remark:
setattr(self, k, v)
elif k in f:
setattr(self, k, _fields[k]._value(v))
f.remove(k)
elif _meta.strict:
raise DataException("trying to set unknown field %s" % k)
else:
setattr(self, k, v)
# add missing fields
for k in f:
_f = _fields[k]
setattr(self, k, _f._value(_f.default))
def scan(self, row):
'''scan row and harvest distinct value.
Takes a row of data and parses the required fields out of this.
Args:
row (list): array of source data
Returns:
parsed and processed value.
Raises:
:class:`~.ValidationException`: raised if explicit validation
fails.
'''
# see if we want to read a column in the row
v = None
if self.pos >= 0:
try:
_v = row[self.pos]
except:
raise DataException(
'parsing %r, row len %d, index %d not found', self.name,
len(row), self.pos)
# do null check if enabled
if self.nullable is not None and _v == self.nullable:
return v
v = _v
if self.validate and not self.validate(v):
raise ValidationException(
'field %r input data did not validate' % self.name)
# apply intermediate function on output, default is stripping
if self.parse:
v = self.parse(v)
elif self.parse:
v = self.parse(row) or v
# delegate to inner function, to reuse this logic
return self._value(v)