def synthesis(prepare_res, slices, job): dw_passed, _ = prepare_res # Using set_slice on a dataset that was written in analysis is not # actually supported, but since it currently works (as long as that # particular slice wasn't written in analysis) let's test it. dw_passed.set_slice(0) dw_passed.write(**{k: v[0] for k, v in test_data.data.items()}) dw_synthesis_split = DatasetWriter(name="synthesis_split", hashlabel="a") dw_synthesis_split.add("a", "int32") dw_synthesis_split.add("b", "unicode") dw_synthesis_split.get_split_write()(1, "a") dw_synthesis_split.get_split_write_list()([2, "b"]) dw_synthesis_split.get_split_write_dict()({"a": 3, "b": "c"}) dw_synthesis_manual = job.datasetwriter(name="synthesis_manual", columns={"sliceno": "int32"}) dw_nonetest = job.datasetwriter(name="nonetest", columns={t: t for t in test_data.data}) for sliceno in range(slices): dw_synthesis_manual.set_slice(sliceno) dw_synthesis_manual.write(sliceno) dw_nonetest.set_slice(sliceno) dw_nonetest.write( **{ k: v[0] if k in test_data.not_none_capable else None for k, v in test_data.data.items() })
def prepare(params): d = datasets.source caption = options.caption % dict(caption=d.caption, hashlabel=options.hashlabel) if len( d.chain(stop_ds={datasets.previous: 'source'}, length=options.length)) == 1: filename = d.filename else: filename = None dws = [] previous = datasets.previous for sliceno in range(params.slices): if options.as_chain and sliceno == params.slices - 1: name = "default" else: name = str(sliceno) dw = DatasetWriter( caption="%s (slice %d)" % (caption, sliceno), hashlabel=options.hashlabel, filename=filename, previous=previous, name=name, for_single_slice=sliceno, ) previous = (params.jobid, name) dws.append(dw) names = [] for n, c in d.columns.items(): # names has to be in the same order as the add calls # so the iterator returns the same order the writer expects. names.append(n) for dw in dws: dw.add(n, c.type) return dws, names, caption, filename
def prepare(params): assert params.slices >= 2, "Hashing won't do anything with just one slice" dws = DotDict() # all the numeric types should hash the same (for values they have in common) for name, hashlabel, typ in ( ("unhashed_manual", None, "int32"), # manually interlaved ("unhashed_split", None, "int64"), # split_write interlaved ("up_checked", "up", "float32"), # hashed on up using dw.hashcheck ("up_split", "up", "float64"), # hashed on up using split_write ("down_checked", "down", "bits32"), # hashed on down using dw.hashcheck ("down_discarded", "down", "bits64"), # hashed on down using discarding writes ("down_discarded_list", "down", "number"), # hashed on down using discarding list writes ("down_discarded_dict", "down", "complex32"), # hashed on down using discarding dict writes # we have too many types, so we need more datasets ("unhashed_complex64", None, "complex64"), ("unhashed_bytes", None, "bytes"), ("up_ascii", "up", "ascii"), ("down_unicode", "down", "unicode"), # datetime on 1970-01-01 hashes like time ("up_datetime", "up", "datetime"), ("down_time", "down", "time"), # date doesn't hash the same as anything else, so compare it to itself ("up_date", "up", "date"), ("down_date", "down", "date"), ): dw = DatasetWriter(name=name, hashlabel=hashlabel) dw.add("up", typ) dw.add("down", typ) dws[name] = dw return dws
def prepare(job, slices): assert slices >= test_data.value_cnt dw_default = DatasetWriter() dw_default.add("a", "number") dw_default.add("b", "ascii") DatasetWriter(name="named", columns={"c": "bool", "d": "date"}) dw_passed = job.datasetwriter(name="passed", columns=test_data.columns) return dw_passed, 42
def prepare(params): dws = {} prev = None for name in "abcdefgh": dw = DatasetWriter(name=name, previous=prev) dw.add("ds", "ascii") dw.add("num", "number") dws[name] = dw prev = dw return dws
def test_filter_bad_with_rename_and_chain(): dw = DatasetWriter(name="filter bad with rename", allow_missing_slices=True) dw.add('a', 'ascii') dw.add('b', 'bytes') dw.add('c', 'unicode') dw.set_slice(0) dw.write('0', b'1', '2') dw.write('9', B'A', 'B') dw.write('C', B'D', 'E') source_ds = dw.finish() jid = subjobs.build( 'dataset_type', column2type=dict(b='int32_10', c='int64_16', d='int32_16'), filter_bad=True, rename=dict(a='b', b='c', c='d'), source=source_ds, ) typed_ds = jid.dataset() coltypes = sorted( (name, col.type) for name, col in typed_ds.columns.items()) assert coltypes == [('b', 'int32'), ('c', 'int64'), ('d', 'int32')], coltypes assert list(typed_ds.iterate(0)) == [(0, 1, 2), (9, 10, 11)] bad_ds = jid.dataset('bad') coltypes = sorted((name, col.type) for name, col in bad_ds.columns.items()) assert coltypes == [('b', 'ascii'), ('c', 'bytes'), ('d', 'unicode')], coltypes assert list(bad_ds.iterate(0)) == [('C', b'D', 'E')] dw = DatasetWriter(name="filter bad with rename chain", allow_missing_slices=True, previous=source_ds) dw.add('a', 'ascii') dw.add('b', 'ascii') dw.add('c', 'ascii') dw.set_slice(0) dw.write('3', '4', '5') dw.write('6', '7', 'eight') source_ds = dw.finish() jid = subjobs.build( 'dataset_type', column2type=dict(a='number', b='int32_10', c='int64_10'), defaults=dict(a='8'), filter_bad=True, rename=dict(a='b', b='c', c='a'), source=source_ds, ) typed_ds = jid.dataset() coltypes = sorted( (name, col.type) for name, col in typed_ds.columns.items()) assert coltypes == [('a', 'number'), ('b', 'int32'), ('c', 'int64')], coltypes assert list(typed_ds.iterate(0)) == [(2, 0, 1), (5, 3, 4), (8, 6, 7)] bad_ds = jid.dataset('bad') coltypes = sorted((name, col.type) for name, col in bad_ds.columns.items()) assert coltypes == [('a', 'unicode'), ('b', 'ascii'), ('c', 'bytes')], coltypes assert list(bad_ds.iterate(0)) == [('B', '9', b'A'), ('E', 'C', b'D')]
def test_column_discarding(): dw = DatasetWriter(name='column discarding') dw.add('a', 'bytes') dw.add('b', 'bytes') dw.add('c', 'bytes') w = dw.get_split_write() w(b'a', b'b', b'c') source = dw.finish() # Discard b because it's not typed ac_implicit = subjobs.build( 'dataset_type', source=source, column2type=dict(a='ascii', c='ascii'), discard_untyped=True, ).dataset() assert sorted(ac_implicit.columns) == ['a', 'c'], '%s: %r' % (ac_implicit, sorted(ac_implicit.columns),) assert list(ac_implicit.iterate(None)) == [('a', 'c',)], ac_implicit # Discard b explicitly ac_explicit = subjobs.build( 'dataset_type', source=source, column2type=dict(a='ascii', c='ascii'), rename=dict(b=None), ).dataset() assert sorted(ac_explicit.columns) == ['a', 'c'], '%s: %r' % (ac_explicit, sorted(ac_explicit.columns),) assert list(ac_explicit.iterate(None)) == [('a', 'c',)], ac_explicit # Discard c by overwriting it with b. Keep untyped b. ac_bASc = subjobs.build( 'dataset_type', source=source, column2type=dict(a='ascii', c='ascii'), rename=dict(b='c'), ).dataset() assert sorted(ac_bASc.columns) == ['a', 'b', 'c'], '%s: %r' % (ac_bASc, sorted(ac_bASc.columns),) assert list(ac_bASc.iterate(None)) == [('a', b'b', 'b',)], ac_bASc # Discard c by overwriting it with b. Also type b as a different type. abc_bASc = subjobs.build( 'dataset_type', source=source, column2type=dict(a='ascii', b='strbool', c='ascii'), rename=dict(b='c'), ).dataset() assert sorted(abc_bASc.columns) == ['a', 'b', 'c'], '%s: %r' % (abc_bASc, sorted(abc_bASc.columns),) assert list(abc_bASc.iterate(None)) == [('a', True, 'b',)], abc_bASc
def test_rehash_with_empty_slices(): dw = DatasetWriter(name='rehash with empty slices', hashlabel='a') dw.add('a', 'ascii') dw.add('b', 'ascii') w = dw.get_split_write() w('a', '42') w('42', 'b') source = dw.finish() hashfunc = typed_writer('int32').hash def verify_hashing(caption, want_values, **kw): ds = subjobs.build('dataset_type', source=source, column2type=dict(a='int32_10'), caption=caption, **kw).dataset() got_values = set() for sliceno in range(g.slices): for got in ds.iterate(sliceno): assert hashfunc(got[0]) % g.slices == sliceno assert got not in got_values got_values.add(got) assert want_values == got_values verify_hashing('with discard', {( 42, 'b', )}, filter_bad=True) # using defaults uses some different code paths verify_hashing('with default=0 (probably two slices)', {( 0, '42', ), ( 42, 'b', )}, defaults=dict(a='0')) verify_hashing('with default=42 (one slice)', {( 42, '42', ), ( 42, 'b', )}, defaults=dict(a='42'))
def synthesis(job): manual_chain = [Dataset(jobids.selfchain, name) for name in "abcdefgh"] manual_abf = [manual_chain[0], manual_chain[1], manual_chain[5]] # build a local abf chain prev = None for ix, ds in enumerate(manual_abf): name = "abf%d" % (ix, ) prev = ds.link_to_here(name, override_previous=prev) manual_abf_data = list(Dataset.iterate_list(None, None, manual_abf)) local_abf_data = list(Dataset(job, "abf2").iterate_chain(None, None)) assert manual_abf_data == local_abf_data # disconnect h, verify there is no chain manual_chain[-1].link_to_here("alone", override_previous=None) assert len(Dataset(job, "alone").chain()) == 1 # check that the original chain is unhurt assert manual_chain == manual_chain[-1].chain() # So far so good, now make a chain long enough to have a cache. prev = None ix = 0 going = True while going: if prev and "cache" in prev._data: going = False name = "longchain%d" % (ix, ) dw = DatasetWriter(name=name, previous=prev) dw.add("ix", "number") dw.get_split_write()(ix) prev = dw.finish() ix += 1 # we now have a chain that goes one past the first cache point full_chain = Dataset(prev).chain() assert "cache" in full_chain[ -2]._data # just to check the above logic is correct assert "cache" not in full_chain[-1]._data # just to be sure.. full_chain[-2].link_to_here("nocache", override_previous=None) full_chain[-1].link_to_here("withcache", override_previous=full_chain[-3]) assert "cache" not in Dataset(job, "nocache")._data assert "cache" in Dataset(job, "withcache")._data # And make sure they both get the right data too. assert list(Dataset(prev).iterate_chain(None, "ix")) == list(range(ix)) assert list(Dataset(job, "nocache").iterate_chain(None, "ix")) == [ix - 2] assert list(Dataset(job, "withcache").iterate_chain( None, "ix")) == list(range(ix - 2)) + [ix - 1]
def prepare(params): assert params.slices >= 2, "Hashing won't do anything with just one slice" dws = DotDict() for name, hashlabel in ( ("unhashed_manual", None), # manually interlaved ("unhashed_split", None), # split_write interlaved ("up_checked", "up"), # hashed on up using dw.hashcheck ("up_split", "up"), # hashed on up using split_write ("down_checked", "down"), # hashed on down using dw.hashcheck ("down_discarded", "down"), # hashed on down using discarding writes ("down_discarded_list", "down"), # hashed on down using discarding list writes ("down_discarded_dict", "down"), # hashed on down using discarding dict writes ): dw = DatasetWriter(name=name, hashlabel=hashlabel) dw.add("up", "int32") dw.add("down", "int32") dws[name] = dw return dws
def test_filter_bad_across_types(): columns = { 'bytes': 'bytes', 'float64': 'bytes', 'int32_10': 'ascii', 'json': 'unicode', 'number:int': 'unicode', 'unicode:utf-8': 'bytes', } # all_good, *values # Make sure all those types (except bytes) can filter other lines, # and be filtered by other lines. And that several filtering values # is not a problem (line 11). data = [ [ True, b'first', b'1.1', '1', '"a"', '001', b'ett', ], [ True, b'second', b'2.2', '2', '"b"', '02', b'tv\xc3\xa5', ], [ True, b'third', b'3.3', '3', '["c"]', '3.0', b'tre', ], [ False, b'fourth', b'4.4', '4', '"d"', '4.4', b'fyra', ], # number:int bad [ False, b'fifth', b'5.5', '-', '"e"', '5', b'fem', ], # int32_10 bad [ False, b'sixth', b'6.b', '6', '"f"', '6', b'sex', ], # float64 bad [ False, b'seventh', b'7.7', '7', '{"g"}', '7', b'sju', ], # json bad [ False, b'eigth', b'8.8', '8', '"h"', '8', b'\xa5\xc3tta', ], # unicode:utf-8 bad [ True, b'ninth', b'9.9', '9', '"i"', '9', b'nio', ], [ True, b'tenth', b'10', '10', '"j"', '10', b'tio', ], [ False, b'eleventh', b'11a', '1-', '"k",', '1,', b'elva', ], # float64, int32_10 and number:int bad [ True, b'twelfth', b'12', '12', '"l"', '12', b'tolv', ], ] dw = DatasetWriter(name="filter bad across types", columns=columns) cols_to_check = ['int32_10', 'bytes', 'json', 'unicode:utf-8'] if PY3: # z so it sorts last. dw.add('zpickle', 'pickle') cols_to_check.append('zpickle') for ix in range(len(data)): data[ix].append({ix}) dw.set_slice(0) want = [] def add_want(ix): v = data[ix] want.append(( int(v[3]), v[1], json.loads(v[4]), v[6].decode('utf-8'), )) if PY3: want[-1] = want[-1] + (v[7], ) for ix, v in enumerate(data): if v[0]: add_want(ix) dw.write(*v[1:]) for sliceno in range(1, g.slices): dw.set_slice(sliceno) source_ds = dw.finish() # Once with just filter_bad, once with some defaults too. defaults = {} for _ in range(2): jid = subjobs.build( 'dataset_type', datasets=dict(source=source_ds), options=dict(column2type={t: t for t in columns}, filter_bad=True, defaults=defaults), ) typed_ds = Dataset(jid) got = list(typed_ds.iterate(0, cols_to_check)) assert got == want, "Exptected %r, got %r from %s (from %r%s)" % ( want, got, typed_ds, source_ds, ' with defaults' if defaults else '') # make more lines "ok" for the second lap defaults = {'number:int': '0', 'float64': '0', 'json': '"replacement"'} add_want(3) add_want(5) data[6][4] = '"replacement"' add_want(6) want.sort() # adding them out of order, int32_10 sorts correctly.
def mk_dw(name, cols, **kw): dw = DatasetWriter(name=name, **kw) for colname in cols: dw.add(colname, "unicode") return dw
def prepare(job, slices): # use 256 as a marker value, because that's not a possible char value (assuming 8 bit chars) lf_char = char2int("newline", 256) # separator uses lf_char or \n as the empty value, because memchr might mishandle 256. separator = char2int("separator", 10 if lf_char == 256 else lf_char) comment_char = char2int("comment", 256) if options.quotes == 'True': quote_char = 256 elif options.quotes == 'False': quote_char = 257 else: quote_char = char2int("quotes", 257, "True/False/empty") filename = os.path.join(job.source_directory, options.filename) orig_filename = filename assert 1 <= options.compression <= 9 fds = [os.pipe() for _ in range(slices)] read_fds = [t[0] for t in fds] write_fds = [t[1] for t in fds] if options.labelsonfirstline: labels_rfd, labels_wfd = os.pipe() else: labels_wfd = -1 success_rfd, success_wfd = os.pipe() status_rfd, status_wfd = os.pipe() p = Process(target=reader_process, name="reader", args=(slices, filename, write_fds, labels_wfd, success_wfd, status_wfd, comment_char, lf_char)) p.start() for fd in write_fds: os.close(fd) os.close(success_wfd) os.close(status_wfd) if options.labelsonfirstline: os.close(labels_wfd) # re-use import logic out_fns = ["labels"] r_num = cstuff.mk_uint64(3) res = cstuff.backend.import_slice(*cstuff.bytesargs(labels_rfd, -1, -1, -1, out_fns, b"wb1", separator, r_num, quote_char, lf_char, 0)) os.close(labels_rfd) assert res == 0, "c backend failed in label parsing" with typed_reader("bytes")("labels") as fh: labels_from_file = [lab.decode("utf-8", "backslashreplace") for lab in fh] os.unlink("labels") else: labels_from_file = None labels = options.labels or labels_from_file assert labels, "No labels" labels = [options.rename.get(x, x) for x in labels] assert '' not in labels, "Empty label for column %d" % (labels.index(''),) assert len(labels) == len(set(labels)), "Duplicate labels: %r" % (labels,) dw = DatasetWriter( columns={n: 'bytes' for n in labels if n not in options.discard}, filename=orig_filename, caption='csvimport of ' + orig_filename, previous=datasets.previous, meta_only=True, ) if options.lineno_label: dw.add(options.lineno_label, "int64") if options.allow_bad: bad_dw = DatasetWriter( name="bad", columns=dict(lineno="int64", data="bytes"), caption='bad lines from csvimport of ' + orig_filename, meta_only=True, ) else: bad_dw = None if options.comment or options.skip_lines: skipped_dw = DatasetWriter( name="skipped", columns=dict(lineno="int64", data="bytes"), caption='skipped lines from csvimport of ' + orig_filename, meta_only=True, ) else: skipped_dw = None return separator, quote_char, lf_char, filename, orig_filename, labels, dw, bad_dw, skipped_dw, read_fds, success_rfd, status_rfd,
def prepare(): dw = DatasetWriter() dw.add("str", "ascii") dw.add("num", "number") return dw