def main(): fname = 'foo.dat' N = 100000 with Timer() as t: write_data(N, fname) print "=> write_data: %s s" % t.secs with Timer() as t: test_write(N, fname) print "=> test_write(): %s s" % t.secs with Timer() as t: read_data(fname) print "=> read_data: %s s" % t.secs with Timer() as t: read_data(fname, 100000) print "=> read_data(100000): %s s" % t.secs with Timer() as t: read_data(fname, 50000) print "=> read_data(50000): %s s" % t.secs with open(fname, 'rb', buffering=(4096 * 4)) as f: reader = BinaryDownStreamFilter(f) for i in range(10): cmd, args = reader.next() print cmd, args
def main(): fname = 'foo.dat' N = 100000 with Timer() as t: write_data(N, fname) print "=> write_data: %s s" % t.secs with Timer() as t: test_write(N, fname) print "=> test_write(): %s s" % t.secs with Timer() as t: read_data(fname) print "=> read_data: %s s" % t.secs with Timer() as t: read_data(fname, 100000) print "=> read_data(100000): %s s" % t.secs with Timer() as t: read_data(fname, 50000) print "=> read_data(50000): %s s" % t.secs with open(fname, 'rb', buffering=(4096*4)) as f: reader = BinaryDownStreamFilter(f) for i in range(10): cmd, args = reader.next() print cmd, args
def read_data(fname, N=None): with open(fname, 'rb', buffering=(4096 * 4)) as f: reader = BinaryDownStreamFilter(f) if N is None: for cmd, args in reader: pass else: for i in range(N): cmd, args = reader.next()
def read_data(fname, N=None): with open(fname, 'rb', buffering=(4096*4)) as f: reader = BinaryDownStreamFilter(f) if N is None: for cmd, args in reader: pass else: for i in range(N): cmd, args = reader.next()
def test_digest(self): with open(JOB_TOKEN) as f: f.read(4) # magic deserialize(int, f) # prot deserialize(int, f) # n deserialize(str, f) # label deserialize(str, f) # job passwd = deserialize(str, f) with open(MAP_JAVA_DOWNLINK_DATA) as istream: cmd_stream = BinaryDownStreamFilter(istream) cmd, args = cmd_stream.next() self.assertEqual(cmd, 'authenticationReq') xdigest = '5bMR7RdwmkLvK582eYWEK8X6jDA=' xchallenge = '1593317824749889452062285518813742155' digest, challenge = args self.assertEqual(digest, xdigest) self.assertEqual(challenge, xchallenge) self.assertEqual(digest, create_digest(passwd, challenge))
def test_downlink(self): fname = self._mkfn('foo.bin') stream_writer(fname, STREAM_1) with open(fname, 'r') as f: stream = BinaryDownStreamFilter(f) try: for (cmd, args), vals in it.izip(stream, STREAM_1): self.assertEqual(cmd, vals[0]) self.assertTrue((len(vals) == 1 and not args) or (vals[1:] == args)) except ProtocolError as e: print 'error -- %s' % e
def __run_test(self, mode, mapper_class, context_class): cmd_file = self.__write_cmd_file(mode) pp.run_task( pp.Factory(mapper_class=mapper_class), private_encoding=False, context_class=context_class, cmd_file=cmd_file ) out_fn = cmd_file + '.out' out_records = [] with open(out_fn) as ostream: for cmd, args in BinaryDownStreamFilter(ostream): if cmd == 'output': name, color = args out_records.append({'name': name, 'favorite_color': color}) self.assertEqual(len(out_records), len(self.records)) for out_r, r in zip(out_records, self.records): for k, v in out_r.iteritems(): self.assertEqual(v, r[k])
def _test_map_reduce_with_private_encoding_helper(self, factory, fast_combiner=False): self.stream3.close() cmd_file = self.stream3.name out_file = cmd_file + '.out' reduce_infile = cmd_file + '.reduce' reduce_outfile = reduce_infile + '.out' run_task(factory, cmd_file=cmd_file, private_encoding=True, fast_combiner=fast_combiner) data = {} with open(out_file) as f: bf = BinaryDownStreamFilter(f) for cmd, args in bf: if cmd == 'output': data.setdefault(args[0], []).append(args[1]) stream = [] stream.append(('start', 0)) stream.append(('setJobConf', ('key1', 'value1', 'key2', 'value2'))) stream.append(('runReduce', 0, False)) for k in data: stream.append(('reduceKey', k)) for v in data[k]: stream.append(('reduceValue', v)) stream.append(('close', )) binary_stream_writer(reduce_infile, stream) run_task(factory, cmd_file=reduce_infile, private_encoding=True) with open(reduce_outfile) as f, self._mkf('foo.out', mode='w') as o: bf = BinaryUpStreamDecoder(f) for cmd, args in bf: if cmd == 'progress': o.write('progress\t%s\n' % args[0]) elif cmd == 'output': o.write('output\t%s\n' % '\t'.join(args)) elif cmd == 'done': o.write('done\n') self.check_result('foo.out', STREAM_3)
def decode(istream, ostream): cmd_stream = BinaryDownStreamFilter(istream) for (cmd, args) in cmd_stream: ostream.write('cmd: {}, args: {}\n'.format(cmd, args))
def read_data(fname): with open(fname, 'rb', buffering=(4096 * 4)) as f: reader = BinaryDownStreamFilter(f) for cmd, args in reader: pass