def test_get_key_values_stream2(self): stream = get_stream(STREAM_2_DATA) kvs_stream = get_key_values_stream(stream, private_encoding=False) kvs2_stream = it.groupby(STREAM_2_DATA, keyfunc) for (k1, vals1), (k2, itx) in czip(kvs_stream, kvs2_stream): self.assertEqual(k1, k2) if k1 == 'key2': vals2 = cmap(lambda t: t[1], cfilter(lambda t: t[0] == streams.REDUCE_VALUE, itx)) for v1, v2 in czip(vals1, vals2): self.assertEqual(v1, v2)
def test_avro_reader(self): N = 500 fn = self.write_avro_file(avro_user_record, N, 1024) url = hdfs.path.abspath(fn, local=True) class FunkyCtx(object): def __init__(self, isplit): self.input_split = isplit def get_areader(offset, length): isplit = InputSplit(InputSplit.to_string(url, offset, length)) ctx = FunkyCtx(isplit) return AvroReader(ctx) areader = get_areader(0, 14) file_length = areader.reader.file_length with self.assertRaises(StopIteration): next(areader) areader = get_areader(0, file_length) with SeekableDataFileReader(open(fn, 'rb'), DatumReader()) as sreader: for (o, a), s in czip(areader, sreader): self.assertEqual(a, s) mid_len = int(file_length / 2) lows = [x for x in get_areader(0, mid_len)] highs = [x for x in get_areader(mid_len, file_length)] self.assertEqual(N, len(lows) + len(highs))
def test_seekable(self): fn = self.write_avro_file(avro_user_record, 500, 1024) with open(fn, 'rb') as f: sreader = SeekableDataFileReader(f, DatumReader()) res = [t for t in czip(cmap( lambda _: f.tell(), it.repeat(1) ), sreader)] sreader.align_after(res[-1][0]) with self.assertRaises(StopIteration): r = next(sreader) sreader.align_after(0) r = next(sreader) self.assertEqual(r, res[0][1]) def offset_iterator(): s = -1 for o, r in res: sreader.align_after(o) t = f.tell() if t == s: continue s = t x = next(sreader) yield (t, x) i = 0 for xo, x in offset_iterator(): sreader.align_after(xo) for o, r in res[i:]: if o >= xo: self.assertEqual(x, r) break i += 1
def link_helper(self, mod, Writer, DownStreamAdapter): fname = self._mkfn('foo.' + ('bin' if mod == 'b' else 'txt')) stream_writer(fname, STREAM_1, mod, Writer) with open(fname, 'r' + mod) as f: stream = DownStreamAdapter(f) try: at_least_once_in_loop = False for (cmd, args), vals in czip(stream, STREAM_1): at_least_once_in_loop = True self.assertEqual(cmd, vals[0]) vals = vals[1:] if mod == 'b': vals = [ x.encode('utf-8') if isinstance(x, unicode) else x for x in vals ] vals = tuple(vals) if cmd == streams.SET_JOB_CONF: self.assertEqual(len(args), 1) self.assertEqual(args[0], vals) else: self.assertTrue((len(vals) == 0 and not args) or (vals == args)) self.assertTrue(at_least_once_in_loop) except ProtocolError as e: print('error -- %s' % e)
def link_helper(self, mod, Writer, DownStreamAdapter): fname = self._mkfn('foo.' + ('bin' if mod == 'b' else 'txt')) stream_writer(fname, STREAM_1, mod, Writer) with open(fname, 'r' + mod) as f: stream = DownStreamAdapter(f) try: at_least_once_in_loop = False for (cmd, args), vals in czip(stream, STREAM_1): at_least_once_in_loop = True self.assertEqual(cmd, vals[0]) vals = vals[1:] if mod == 'b': vals = [x.encode('utf-8') if isinstance(x, unicode) else x for x in vals] vals = tuple(vals) if cmd == streams.SET_JOB_CONF: self.assertEqual(len(args), 1) self.assertEqual(args[0], vals) else: self.assertTrue((len(vals) == 0 and not args) or (vals == args)) self.assertTrue(at_least_once_in_loop) except ProtocolError as e: print('error -- %s' % e)
def __cp_recursive(self, wd): src_t = self.__make_tree(wd) src = src_t.name copy_on_wd = "%s_copy" % src src_bn, copy_on_wd_bn = [ hdfs.path.basename(d) for d in (src, copy_on_wd) ] hdfs.cp(src, copy_on_wd, mode="wb") exp_t = self.__make_tree(wd, root=copy_on_wd_bn, create=False) for t, exp_t in czip(src_t.walk(), exp_t.walk()): self.assertTrue(hdfs.path.exists(exp_t.name)) if t.kind == 0: self.assertEqual(hdfs.load(exp_t.name), self.data) # check semantics when target dir already exists hdfs.rmr(copy_on_wd) hdfs.mkdir(copy_on_wd) hdfs.cp(src, copy_on_wd, mode="wb") exp_t = self.__make_tree(copy_on_wd, root=src_bn, create=False) for t, exp_t in czip(src_t.walk(), exp_t.walk()): self.assertTrue(hdfs.path.exists(exp_t.name)) if t.kind == 0: self.assertEqual(hdfs.load(exp_t.name), self.data)
def __cp_recursive(self, wd): src_t = self.__make_tree(wd) src = src_t.name copy_on_wd = "%s_copy" % src src_bn, copy_on_wd_bn = [ hdfs.path.basename(d) for d in (src, copy_on_wd) ] hdfs.cp(src, copy_on_wd, mode="wb") exp_t = self.__make_tree(wd, root=copy_on_wd_bn, create=False) for t, exp_t in czip(src_t.walk(), exp_t.walk()): self.assertTrue(hdfs.path.exists(exp_t.name)) if t.kind == 0: self.assertEqual(hdfs.load(exp_t.name), self.data) # check semantics when target dir already exists hdfs.rmr(copy_on_wd) hdfs.mkdir(copy_on_wd) hdfs.cp(src, copy_on_wd, mode="wb") exp_t = self.__make_tree(copy_on_wd, root=src_bn, create=False) for t, exp_t in czip(src_t.walk(), exp_t.walk()): self.assertTrue(hdfs.path.exists(exp_t.name)) if t.kind == 0: self.assertEqual(hdfs.load(exp_t.name), self.data)
def __ls(self, ls_func, path_transform): for wd, paths in czip((self.local_wd, self.hdfs_wd), (self.local_paths, self.hdfs_paths)): for p in paths: hdfs.dump(self.data, p, mode="wb") test_dir = "%s/%s" % (wd, "test_dir") test_path = "%s/%s" % (test_dir, "test_path") hdfs.dump(self.data, test_path, mode="wb") paths.append(test_dir) for recursive in False, True: if recursive: paths.append(test_path) dir_list = [ path_transform(p) for p in ls_func(wd, recursive=recursive) ] self.assertEqual(sorted(dir_list), sorted(paths))
def __ls(self, ls_func, path_transform): for wd, paths in czip( (self.local_wd, self.hdfs_wd), (self.local_paths, self.hdfs_paths) ): for p in paths: hdfs.dump(self.data, p, mode="wb") test_dir = "%s/%s" % (wd, "test_dir") test_path = "%s/%s" % (test_dir, "test_path") hdfs.dump(self.data, test_path, mode="wb") paths.append(test_dir) for recursive in False, True: if recursive: paths.append(test_path) dir_list = [ path_transform(p) for p in ls_func(wd, recursive=recursive) ] self.assertEqual(sorted(dir_list), sorted(paths))
def test_get_key_value_stream(self): stream = get_stream(STREAM_1_DATA) kv_stream = get_key_value_stream(stream) for ((k, v), (cmd, k1, v1)) in czip(kv_stream, STREAM_1_DATA): self.assertEqual(k, k1) self.assertEqual(v, v1)