def emit(self, v, escaper=None):
'''helper function to export this field.
Expects a value from the model to be emitted
Args:
v: value to emit
escaper: escaper function to apply on value
Returns:
emitted value.
Raises:
:class:`~.ValidationException`: raised if explicit validation
fails.'''
if self.max_length and isstr(v):
v = v[:self.max_length]
if v is None:
v = self.default if self.default is not None else v
if self.validate_output and not self.validate_output(v):
raise ValidationException("not able to validate %s=%s" %
(self.name, v))
# allow external function (e.g. SQL escape)
# anonymize this data
if self.anonymize:
v = self.anonymize(v)
# check if we have a replacement string to take into account
if self.replace:
if not isinstance(v, tuple):
v = (v, )
v = self.replace(v) # pylint: disable=not-callable
elif escaper:
v = escaper(v)
return v
def _value(self, v):
if isstr(v):
if v == "":
return None
try:
v = p.parse(v)
except ValueError:
raise DataException("%s could not parse date %s", self.name, v)
return v
def __init__(self,
pos=-1,
name="",
default=None,
nullable="NULL",
key=False,
required=False,
replacement=None,
parse=None,
validate=None,
anonymize=None,
max_length=None,
unique=False,
validate_output=None):
# default value if null
self.default = default if default is not None else getattr(
self.__class__, 'default', default)
# key indicated key field
self.key = key
# fixed position in the row to read
if max_length and self.schema_type != "string":
raise DefinitionException("Cannot set max_length on on string")
self.max_length = max_length if isinstance(max_length, int) else None
# name of this field (will be set in Model class construction)
self.name = name
# input string that defines null -> None
self.nullable = nullable
# some function to apply to value
self.parse = parse or getattr(self.__class__, 'parse', None)
self.pos = int(pos)
# replace string to use in output
if isstr(replacement):
replacement = partial(_replace, replacement)
self.replace = getattr(self.__class__, 'replace', replacement)
# required indicates must be filled in
self.required = required
# unique indicates a unique field
self.unique = unique
# anonymize is the anonymization function
self.anonymize = anonymize() if isinstance(anonymize, type) \
else anonymize
# some function to apply to value
self.validate = validate or getattr(self.__class__, 'validate', None)
# output validator
self.validate_output = validate_output
# creation_order is required for orderdict to retain order of fields
self.creation_order = BaseField.creation_order
BaseField.creation_order += 1
def _value(self, v):
return v.strip() if isstr(v) else v
def _value(self, v):
return int(v) if isstr(v) else v
def read_map_from_csv(key=0,
value=1,
f=None,
delimiter="\t",
header=True,
as_list=False,
unique=False):
'''Generates a map from a csv and adds some validation and list parsing. A
function that returns a map for MappingField to use as input in its
MappingField.data_map.
>>> from data_migrator.contrib.read import read_map_from_csv
>>> table_map = read_map_from_csv(f=open('table.csv'), delimiter=';',
key='id', value='name')
>>> len(table_map)
10
Note that by default it is expected to have headers in the csv.
Args:
f: Filehandle to read the csv from into the map
delimiter: Option to select another delimiter, other than `\\\\t`
key: Name or position of the Key, if ``header`` is false, the ordinal
position is expected (default first)
value: Name or position of the Value, if ``header`` is false, the
ordinal position is expected (default second)
as_list (boolean): If ``True``, *data-migrator* will treat add all
values for ``key`` as a list. Default is ``False``.
unique (boolean): If ``True``, *data-migrator* will treat add all non
unique values for ``key`` as a violation and raise a
:exc:`~.NonUniqueDataException`. Default is ``False``.
header (boolean): If ``True``, *data-migrator* will treat row as a
header column. Default is ``True``
Returns:
map: a key, value map from the csv
Raises:
:exc:`~.DefinitionException`: if key, value does not match or as_list
not set.
:exc:`~.NonUniqueDataException`: if data is not unique on the key.
'''
data_map = {}
if not f:
f = sys.stdin
r = csv.reader(f, delimiter=delimiter)
if header:
h = next(r, None)
try:
if isstr(key):
ki = h.index(key)
elif not header:
ki = key
else:
raise DefinitionException('key=%s - should be string' % key)
if isstr(value):
vi = h.index(value)
elif not header:
vi = value
else:
raise DefinitionException("value=%s - should be string" % value)
except ValueError as err:
raise DefinitionException(err)
i = 0
for l in r:
i += 1
v = [l[vi]] if as_list else l[vi]
if l[ki] in data_map:
if unique:
raise NonUniqueDataException(
'line %d - unique constraint failed: %s' % (i, l[ki]))
elif as_list:
data_map[l[ki]] += v
else:
raise DefinitionException(
'line %d - unique contraint failed, expecting as_list for %s:%s'
% (i, l[ki], data_map[l[ki]]))
else:
data_map[l[ki]] = v
if f != sys.stdin:
f.close()
return data_map
