def validate_copy(self, data, columns, pfx=""): """Validate tab-separated fields""" ok, _unicode = skytools.safe_utf8_decode(data) if ok: return data # log error vals = data.split('\t') for i, v in enumerate(vals): ok, tmp = skytools.safe_utf8_decode(v) if not ok: self.show_error(columns[i], v, pfx, tmp) # return safe data return _unicode.encode('utf8')
def validate_string(self, value, pfx=""): """validate string""" ok, u = skytools.safe_utf8_decode(value) if ok: return value _pfx = pfx and (pfx+': ') or "" self.log.info('%sFixed invalid UTF8 in string <%s>', _pfx, value) return u.encode('utf8')
def validate_string(self, value, pfx=""): """validate string""" ok, u = skytools.safe_utf8_decode(value) if ok: return value _pfx = pfx and (pfx + ': ') or "" self.log.info('%sFixed invalid UTF8 in string <%s>', _pfx, value) return u.encode('utf8')
def validate_dict(self, data, pfx=""): """validates data in dict""" for k, v in data.items(): if v: ok, u = skytools.safe_utf8_decode(v) if not ok: self.show_error(k, v, pfx, u) data[k] = u.encode('utf8') return data