def analysis_lap(vars):
if vars.rehashing:
if vars.first_lap:
out_fn = 'hashtmp.%d' % (vars.sliceno, )
colname = vars.rev_rename.get(vars.dw.hashlabel, vars.dw.hashlabel)
coltype = vars.column2type[options.rename.get(colname, colname)]
vars.rehashing = False
real_coltype = one_column(vars, colname, coltype, [out_fn], True)
vars.rehashing = True
assert vars.res_bad_count[colname] == [0
] # imlicitly has a default
vars.slicemap_fd = map_init(vars, 'slicemap%d' % (vars.sliceno, ),
'slicemap_size')
slicemap = mmap(vars.slicemap_fd, vars.slicemap_size)
slicemap = Int16BytesWrapper(slicemap)
hash = typed_writer(real_coltype).hash
slices = vars.slices
vars.hash_lines = hash_lines = [0] * slices
for ix, value in enumerate(typed_reader(real_coltype)(out_fn)):
dest_slice = hash(value) % slices
slicemap[ix] = dest_slice
hash_lines[dest_slice] += 1
unlink(out_fn)
for colname, coltype in vars.column2type.items():
if vars.rehashing:
out_fns = [
vars.dw.column_filename(colname, sliceno=s)
for s in range(vars.slices)
]
else:
out_fns = [vars.dw.column_filename(colname)]
one_column(vars, vars.rev_rename.get(colname, colname), coltype,
out_fns)
return vars.res_bad_count, vars.res_default_count, vars.res_minmax
def analysis(sliceno, params):
assert list(datasets.source.iterate(sliceno, "a")) == [sliceno, 42]
assert list(datasets.source.iterate(sliceno, "b")) == ["a", str(sliceno)]
named = Dataset(datasets.source, "named")
assert list(named.iterate(sliceno, "c")) == [True, False]
assert list(named.iterate(sliceno, "d")) == [
date(1536, 12, min(sliceno + 1, 31)),
date(2236, 5, min(sliceno + 1, 31))
]
if sliceno < test_data.value_cnt:
passed = Dataset(datasets.source, "passed")
good = tuple(v[sliceno] for _, v in sorted(test_data.data.items()))
assert list(passed.iterate(sliceno)) == [good]
synthesis_split = Dataset(datasets.source, "synthesis_split")
values = zip((
1,
2,
3,
), "abc")
hash = typed_writer("int32").hash
good = [v for v in values if hash(v[0]) % params.slices == sliceno]
assert list(synthesis_split.iterate(sliceno)) == good
synthesis_manual = Dataset(datasets.source, "synthesis_manual")
assert list(synthesis_manual.iterate(sliceno, "sliceno")) == [sliceno]
nonetest = Dataset(datasets.source, "nonetest")
good = tuple(v[0] if k in test_data.not_none_capable else None
for k, v in sorted(test_data.data.items()))
assert list(nonetest.iterate(sliceno)) == [good]
def verify(slices, data, source, previous=None, **options):
jid = subjobs.build(
"dataset_hashpart",
datasets=dict(source=source, previous=previous),
options=options,
)
hl = options["hashlabel"]
h = typed_writer(columns[hl][0]).hash
ds = Dataset(jid)
good = {row[hl]: row for row in data}
names = list(source.columns)
for slice in range(slices):
for row in ds.iterate_chain(slice, names):
row = dict(zip(names, row))
assert h(
row[hl]
) % slices == slice, "row %r is incorrectly in slice %d in %s" % (
row, slice, ds)
want = good[row[hl]]
assert row == want, '%s (rehashed from %s) did not contain the right data for "%s".\nWanted\n%r\ngot\n%r' % (
ds, source, hl, want, row)
return ds
def one_column(vars, colname, coltype, out_fns, for_hasher=False):
if for_hasher:
record_bad = skip_bad = False
elif vars.first_lap:
record_bad = options.filter_bad
skip_bad = False
else:
record_bad = 0
skip_bad = options.filter_bad
minmax_fn = 'minmax%d' % (vars.sliceno, )
fmt = fmt_b = None
is_null_converter = False
if coltype in dataset_type.convfuncs:
shorttype = coltype
_, cfunc, pyfunc = dataset_type.convfuncs[coltype]
elif coltype.startswith('null_'):
shorttype = coltype
pyfunc = False
cfunc = True
is_null_converter = True
else:
shorttype, fmt = coltype.split(':', 1)
_, cfunc, pyfunc = dataset_type.convfuncs[shorttype + ':*']
if cfunc:
cfunc = shorttype.replace(':', '_')
if pyfunc:
tmp = pyfunc(coltype)
if callable(tmp):
pyfunc = tmp
cfunc = None
else:
pyfunc = None
cfunc, fmt, fmt_b = tmp
if coltype == 'number':
cfunc = 'number'
elif coltype == 'number:int':
coltype = 'number'
cfunc = 'number'
fmt = "int"
assert cfunc or pyfunc, coltype + " didn't have cfunc or pyfunc"
coltype = shorttype
in_fns = []
offsets = []
max_counts = []
for d in vars.chain:
assert colname in d.columns, '%s not in %s' % (
colname,
d,
)
if not is_null_converter:
assert d.columns[
colname].type in byteslike_types, '%s has bad type in %s' % (
colname,
d,
)
in_fns.append(d.column_filename(colname, vars.sliceno))
if d.columns[colname].offsets:
offsets.append(d.columns[colname].offsets[vars.sliceno])
max_counts.append(d.lines[vars.sliceno])
else:
offsets.append(0)
max_counts.append(-1)
if cfunc:
default_value = options.defaults.get(colname, cstuff.NULL)
if for_hasher and default_value is cstuff.NULL:
if coltype.startswith('bits'):
# No None-support.
default_value = '0'
else:
default_value = None
default_len = 0
if default_value is None:
default_value = cstuff.NULL
default_value_is_None = True
else:
default_value_is_None = False
if default_value != cstuff.NULL:
if isinstance(default_value, unicode):
default_value = default_value.encode("utf-8")
default_len = len(default_value)
c = getattr(cstuff.backend, 'convert_column_' + cfunc)
if vars.rehashing:
c_slices = vars.slices
else:
c_slices = 1
bad_count = cstuff.mk_uint64(c_slices)
default_count = cstuff.mk_uint64(c_slices)
gzip_mode = "wb%d" % (options.compression, )
safe_to_skip_write = vars.rehashing and not options.as_chain
res = c(*cstuff.bytesargs(
in_fns, len(in_fns), out_fns, gzip_mode, minmax_fn, default_value,
default_len, default_value_is_None, fmt, fmt_b, record_bad,
skip_bad, vars.badmap_fd, vars.badmap_size, c_slices,
vars.slicemap_fd, vars.slicemap_size, bad_count, default_count,
offsets, max_counts, safe_to_skip_write))
assert not res, 'Failed to convert ' + colname
vars.res_bad_count[colname] = list(bad_count)
vars.res_default_count[colname] = sum(default_count)
coltype = coltype.split(':', 1)[0]
if is_null_converter:
real_coltype = vars.chain[0].columns[colname].backing_type
mins = []
maxs = []
# Some lines may have been filtered out, so these minmax values
# could be wrong. There's no easy/cheap way to fix that though,
# and they will never be wrong in the bad direction.
for d in vars.chain:
col = d.columns[colname]
if col.min is not None:
mins.append(col.min)
maxs.append(col.max)
if mins:
vars.res_minmax[colname] = [min(mins), max(maxs)]
else:
real_coltype = dataset_type.typerename.get(coltype, coltype)
with type2iter[real_coltype](minmax_fn) as it:
vars.res_minmax[colname] = list(it)
unlink(minmax_fn)
else:
# python func
if for_hasher:
raise Exception("Can't hash on column of type %s." % (coltype, ))
nodefault = object()
if colname in options.defaults:
default_value = options.defaults[colname]
if default_value is not None:
if isinstance(default_value, unicode):
default_value = default_value.encode('utf-8')
default_value = pyfunc(default_value)
else:
default_value = nodefault
if options.filter_bad:
badmap = mmap(vars.badmap_fd, vars.badmap_size)
if PY2:
badmap = IntegerBytesWrapper(badmap)
if vars.rehashing:
slicemap = mmap(vars.slicemap_fd, vars.slicemap_size)
slicemap = Int16BytesWrapper(slicemap)
bad_count = [0] * vars.slices
else:
bad_count = [0]
chosen_slice = 0
default_count = 0
dont_minmax_types = {
'bytes', 'ascii', 'unicode', 'json', 'complex32', 'complex64'
}
real_coltype = dataset_type.typerename.get(coltype, coltype)
do_minmax = real_coltype not in dont_minmax_types
fhs = [typed_writer(real_coltype)(fn) for fn in out_fns]
write = fhs[0].write
col_min = col_max = None
it = itertools.chain.from_iterable(
d._column_iterator(vars.sliceno, colname, _type='bytes')
for d in vars.chain)
for ix, v in enumerate(it):
if vars.rehashing:
chosen_slice = slicemap[ix]
write = fhs[chosen_slice].write
if skip_bad:
if badmap[ix // 8] & (1 << (ix % 8)):
bad_count[chosen_slice] += 1
continue
try:
v = pyfunc(v)
except ValueError:
if default_value is not nodefault:
v = default_value
default_count += 1
elif record_bad:
bad_count[chosen_slice] += 1
bv = badmap[ix // 8]
badmap[ix // 8] = bv | (1 << (ix % 8))
continue
else:
raise Exception("Invalid value %r with no default in %s" %
(
v,
colname,
))
if do_minmax and not isinstance(v, NoneType):
if col_min is None:
col_min = col_max = v
if v < col_min: col_min = v
if v > col_max: col_max = v
write(v)
for fh in fhs:
fh.close()
if vars.rehashing:
slicemap.close()
if options.filter_bad:
badmap.close()
vars.res_bad_count[colname] = bad_count
vars.res_default_count[colname] = default_count
vars.res_minmax[colname] = [col_min, col_max]
return real_coltype
def synthesis(job, slices):
# Test keeping untyped columns.
dw = job.datasetwriter(name='a',
columns={
'a': 'unicode',
'b': ('bytes', True),
'c': ('ascii', True),
'd': ('number', True)
})
write = dw.get_split_write()
write('A', None, None, None)
write('a', b'b', 'c', 0)
a = dw.finish()
assert a.hashlabel == None
typed_a = subjobs.build('dataset_type',
options=dict(hashlabel='a',
column2type={'a': 'ascii'}),
datasets=dict(source=a)).dataset()
assert typed_a.hashlabel == 'a'
assert set(typed_a.iterate(None)) == {('A', None, None, None),
('a', b'b', 'c', 0)}, typed_a
# Test hashing on a column not explicitly typed.
dw = job.datasetwriter(name='b',
columns={
'a': 'unicode',
'b': 'ascii',
'c': 'bytes',
'd': 'unicode'
},
previous=a)
write = dw.get_split_write()
write('A', 'B', b'C', '1')
b = dw.finish()
assert b.hashlabel == None
typed_b = subjobs.build('dataset_type',
options=dict(hashlabel='a',
column2type={'b': 'ascii'}),
datasets=dict(source=b)).dataset()
assert typed_b.hashlabel == 'a'
assert set(typed_b.iterate(None)) == {('a', 'b'), ('A', None),
('A', 'B')}, typed_b
# Test renaming over the original hashlabel
dw = job.datasetwriter(name='c',
columns={
'a': 'unicode',
'b': 'ascii',
'c': 'bytes',
'd': 'unicode'
},
hashlabel='a')
write = dw.get_split_write()
write('\xe5', 'b', b'c', '0')
c = dw.finish()
assert c.hashlabel == 'a'
typed_c = subjobs.build('dataset_type',
options=dict(column2type={
'a': 'ascii',
'd': 'number'
},
rename={'c': 'a'}),
datasets=dict(source=c)).dataset()
assert typed_c.hashlabel == None
assert list(typed_c.iterate(None)) == [('c', 'b', b'c', 0)], typed_c
# Test using the original names but for different columns (keeping hashlabel under new name)
dw = job.datasetwriter(name='d',
columns={
'a': 'unicode',
'b': 'ascii',
'c': 'bytes',
'd': 'unicode'
},
hashlabel='a')
write = dw.get_split_write()
write('\xc5', 'B', B'C', '1')
d = dw.finish()
assert d.hashlabel == 'a'
typed_d = subjobs.build('dataset_type',
options=dict(column2type={
'a': 'bytes',
'b': 'ascii',
'c': 'int32_10',
'd': 'bytes'
},
rename={
'b': 'a',
'c': 'b',
'd': 'c',
'a': 'd'
}),
datasets=dict(source=d)).dataset()
assert typed_d.hashlabel == 'd'
assert list(typed_d.iterate(None)) == [(b'B', 'C', 1, b'\xc3\x85')
], typed_c
# Test various types for hashing and discarding of bad lines.
for hl in (None, 'a', 'b', 'c'):
dw = job.datasetwriter(name='hashed on %s' % (hl, ),
columns={
'a': 'unicode',
'b': 'unicode',
'c': 'unicode'
},
hashlabel=hl)
w = dw.get_split_write()
for ix in range(1000):
w(unicode(ix), '%d.%d' % (ix, ix % 5 == 0),
('{"a": %s}' if ix % 3 else '%d is bad') % (ix, ))
src_ds = dw.finish()
assert src_ds.hashlabel == hl
test(
src_ds,
dict(column2type={
'a': 'int32_10',
'b': 'number:int'
},
filter_bad=True), 800)
test(
src_ds,
dict(column2type={
'a': 'int64_10',
'b': 'number',
'c': 'json'
},
filter_bad=True), 666)
test(
src_ds,
dict(column2type={
'a': 'floatint32ei',
'b': 'number:int',
'c': 'json'
},
filter_bad=True), 533)
test(
src_ds,
dict(column2type={
'from_a': 'number',
'from_b': 'float64',
'from_c': 'ascii'
},
rename=dict(a='from_a', b='from_b', c='from_c')), 1000)
test(
src_ds,
dict(column2type={
'c': 'bits32_16',
'a': 'float32',
'b': 'bytes'
},
rename=dict(a='c', b='a', c='b')), 1000)
# this doesn't test as many permutations, it's just to test more column types.
dw = job.datasetwriter(name='more types')
cols = {
'floatbooli':
cycle(['1.42 or so', '0 maybe', '1 (exactly)']),
'datetime:%Y%m%d %H:%M': [
'2019%02d%02d 17:%02d' % (t % 12 + 1, t % 28 + 1, t % 60)
for t in range(1000)
],
'date:%Y%m%d':
['2019%02d%02d' % (
t % 12 + 1,
t % 28 + 1,
) for t in range(1000)],
'time:%H:%M': ['%02d:%02d' % (t // 60, t % 60) for t in range(1000)],
'timei:%H:%M': [
'%02d:%02d%c' % (t // 60, t % 60, chr(t % 26 + 65))
for t in range(1000)
],
}
gens = []
for coltype, gen in cols.items():
dw.add(coltype.split(':')[0], 'ascii')
gens.append(iter(gen))
dw.add('half', 'bytes')
gens.append(cycle([b'1', b'no']))
w = dw.get_split_write()
for _ in range(1000):
w(*map(next, gens))
src_ds = dw.finish()
assert src_ds.hashlabel == None
column2type = {t.split(':')[0]: t for t in cols}
for hl in column2type:
hashed = subjobs.build('dataset_type',
options=dict(column2type=column2type,
hashlabel=hl),
datasets=dict(source=src_ds)).dataset()
assert hashed.hashlabel == hl
unhashed = subjobs.build('dataset_type',
options=dict(column2type=column2type),
datasets=dict(source=src_ds)).dataset()
assert unhashed.hashlabel == None
rehashed = subjobs.build('dataset_hashpart',
options=dict(hashlabel=hl),
datasets=dict(source=unhashed)).dataset()
assert rehashed.hashlabel == hl
assert hashed.lines == rehashed.lines
assert sum(hashed.lines) == 1000
assert set(hashed.columns.keys()) == set(
unhashed.columns.keys()) == set(rehashed.columns.keys())
# and again with a bad column
column2type['half'] = 'float32'
hashed = subjobs.build('dataset_type',
options=dict(column2type=column2type,
hashlabel=hl,
filter_bad=True),
datasets=dict(source=src_ds)).dataset()
assert hashed.hashlabel == hl
unhashed = subjobs.build('dataset_type',
options=dict(column2type=column2type,
filter_bad=True),
datasets=dict(source=src_ds)).dataset()
assert unhashed.hashlabel == None
rehashed = subjobs.build('dataset_hashpart',
options=dict(hashlabel=hl),
datasets=dict(source=unhashed)).dataset()
assert rehashed.hashlabel == hl
del column2type['half']
assert hashed.lines == rehashed.lines
assert sum(hashed.lines) == 500
assert set(hashed.columns.keys()) == set(
unhashed.columns.keys()) == set(rehashed.columns.keys())
# test rehashing on a column we don't type, over all types.
dw = job.datasetwriter(name='rehash all types',
columns={
'2type': ('ascii', True),
'ascii': ('ascii', True),
'bits32': ('bits32', False),
'bits64': ('bits64', False),
'bool': ('bool', True),
'bytes': ('bytes', True),
'date': ('date', True),
'datetime': ('datetime', True),
'float32': ('float32', True),
'float64': ('float64', True),
'int32': ('int32', True),
'int64': ('int64', True),
'json': ('json', True),
'number': ('number', True),
'time': ('time', True),
'unicode': ('unicode', True),
})
write = dw.get_split_write()
data = {
'42':
('ascii string', 100, 1000, True, b'bytes string', date(2019, 12, 11),
datetime(2019, 12, 11, 20, 7, 21), 1.5, 0.00000001, 99, -11, {
"a": "b"
}, 1e100, time(20, 7, 21), 'unicode string'),
None: (None, 0, 0, None, None, None, None, None, None, None, None,
None, None, None, None),
'18': ('ASCII STRING', 111, 1111, False, b'BYTES STRING',
date(1868, 1, 3), datetime(1868, 1, 3, 13, 14,
5), 2.5, -0.0000001, 67, -99,
[42, ".."], 5e100, time(13, 14, 5), 'UNICODE STRING'),
}
write('42', *data['42'])
write(None, *data[None])
write('18', *data['18'])
src_ds = dw.finish()
data['None'] = data.pop(None)
type2type = {
'ascii': 'unicode:ascii',
'bool': 'unicode:ascii',
'date': 'unicode:ascii',
'datetime': 'unicode:ascii',
'time': 'unicode:ascii',
'bits32': 'bits32_10',
'bits64': 'bits64_10',
'bytes': 'bytes',
'float32': 'float32',
'float64': 'float64',
'int32': 'int32_10',
'int64': 'int64_10',
'number': 'number',
'unicode': 'unicode:ascii',
}
for hl, typeas in sorted(type2type.items()):
ds = subjobs.build('dataset_type',
column2type={
'2type': typeas
},
hashlabel=hl,
source=src_ds).dataset()
seen = set()
hl_hash = typed_writer(hl).hash
for sliceno in range(slices):
for line in ds.iterate(sliceno, None):
key = line[0] or None
if isinstance(key, float):
key = int(key)
if isinstance(key, bytes):
key = key.decode('ascii')
else:
key = unicode(key)
assert data.get(key) == line[
1:], "%s (hl %s) didn't have the right data for line %r" % (
ds,
hl,
line[0],
)
hv = line[sorted(src_ds.columns).index(hl)]
assert hl_hash(
hv
) % slices == sliceno, "%s (hl %s) didn't hash %r correctly" % (
ds,
hl,
hv,
)
assert key not in seen, "%s (hl %s) repeated line %s" % (
ds,
hl,
line[0],
)
seen.add(key)
assert seen == {'42', 'None',
'18'}, "%s didn't have all lines (%r)" % (
ds,
seen,
)
def synthesis(prepare_res, params, job, slices):
dws = prepare_res
for dw in (
dws.unhashed_split,
dws.up_split,
):
w = dw.get_split_write_list()
for row in all_data:
w(row)
for dw in dws.values():
dw.finish()
# Verify that the different ways of writing gave the same result
for names in (
("unhashed_split", "unhashed_manual"),
("up_checked", "up_split"),
("down_checked", "down_discarded", "down_discarded_list",
"down_discarded_dict"),
):
dws = {name: job.dataset(name) for name in names}
assert dws == {name: Dataset((params.jobid, name))
for name in names
}, "Old style Dataset((params.jobid, name)) broken"
for sliceno in range(slices):
data = {name: list(dws[name].iterate(sliceno)) for name in names}
good = data[names[0]]
for name in names[1:]:
assert data[
name] == good, "%s doesn't match %s in slice %d" % (
names[0],
name,
sliceno,
)
# Verify that both up and down hashed on the expected column
hash = typed_writer("int32").hash
for colname in ("up", "down"):
ds = job.dataset(colname + "_checked")
for sliceno in range(slices):
for value in ds.iterate(sliceno, colname):
assert hash(
value
) % slices == sliceno, "Bad hashing on %s in slice %d" % (
colname,
sliceno,
)
# Verify that up and down are not the same, to catch hashing
# not actually hashing.
up = list(job.dataset("up_checked").iterate(None))
down = list(job.dataset("down_checked").iterate(None))
assert up != down, "Hashlabel did not change slice distribution"
# And check that the data is still the same.
assert sorted(up) == sorted(
down) == all_data, "Hashed datasets have wrong data"
# Verify that rehashing works.
# (Can't use sliceno None, because that won't rehash, and even if it did
# the order wouldn't match. Order doesn't even match in the rehashed
# individual slices.)
up = job.dataset("up_checked")
down = job.dataset("down_checked")
unhashed = job.dataset("unhashed_manual")
for sliceno in range(slices):
a = list(up.iterate(sliceno))
b = list(down.iterate(sliceno, hashlabel="up", rehash=True))
c = list(unhashed.iterate(sliceno, hashlabel="up", rehash=True))
assert sorted(a) == sorted(b) == sorted(
c), "Rehashing is broken (slice %d)" % (sliceno, )
# And finally verify that we are not allowed to specify the wrong hashlabel
good = True
try:
up.iterate(None, hashlabel="down")
good = False
except AssertionError:
pass
try:
unhashed.iterate(None, hashlabel="down")
good = False
except AssertionError:
pass
assert good, "Iteration allowed on the wrong hashlabel"