def main():
fname = 'foo.dat'
N = 100000
with Timer() as t:
write_data(N, fname)
print "=> write_data: %s s" % t.secs
with Timer() as t:
test_write(N, fname)
print "=> test_write(): %s s" % t.secs
with Timer() as t:
read_data(fname)
print "=> read_data: %s s" % t.secs
with Timer() as t:
read_data(fname, 100000)
print "=> read_data(100000): %s s" % t.secs
with Timer() as t:
read_data(fname, 50000)
print "=> read_data(50000): %s s" % t.secs
with open(fname, 'rb', buffering=(4096 * 4)) as f:
reader = BinaryDownStreamFilter(f)
for i in range(10):
cmd, args = reader.next()
print cmd, args
def main():
fname = 'foo.dat'
N = 100000
with Timer() as t:
write_data(N, fname)
print "=> write_data: %s s" % t.secs
with Timer() as t:
test_write(N, fname)
print "=> test_write(): %s s" % t.secs
with Timer() as t:
read_data(fname)
print "=> read_data: %s s" % t.secs
with Timer() as t:
read_data(fname, 100000)
print "=> read_data(100000): %s s" % t.secs
with Timer() as t:
read_data(fname, 50000)
print "=> read_data(50000): %s s" % t.secs
with open(fname, 'rb', buffering=(4096*4)) as f:
reader = BinaryDownStreamFilter(f)
for i in range(10):
cmd, args = reader.next()
print cmd, args
def read_data(fname, N=None):
with open(fname, 'rb', buffering=(4096 * 4)) as f:
reader = BinaryDownStreamFilter(f)
if N is None:
for cmd, args in reader:
pass
else:
for i in range(N):
cmd, args = reader.next()
def read_data(fname, N=None):
with open(fname, 'rb', buffering=(4096*4)) as f:
reader = BinaryDownStreamFilter(f)
if N is None:
for cmd, args in reader:
pass
else:
for i in range(N):
cmd, args = reader.next()
def test_digest(self):
with open(JOB_TOKEN) as f:
f.read(4) # magic
deserialize(int, f) # prot
deserialize(int, f) # n
deserialize(str, f) # label
deserialize(str, f) # job
passwd = deserialize(str, f)
with open(MAP_JAVA_DOWNLINK_DATA) as istream:
cmd_stream = BinaryDownStreamFilter(istream)
cmd, args = cmd_stream.next()
self.assertEqual(cmd, 'authenticationReq')
xdigest = '5bMR7RdwmkLvK582eYWEK8X6jDA='
xchallenge = '1593317824749889452062285518813742155'
digest, challenge = args
self.assertEqual(digest, xdigest)
self.assertEqual(challenge, xchallenge)
self.assertEqual(digest, create_digest(passwd, challenge))
def test_downlink(self):
fname = self._mkfn('foo.bin')
stream_writer(fname, STREAM_1)
with open(fname, 'r') as f:
stream = BinaryDownStreamFilter(f)
try:
for (cmd, args), vals in it.izip(stream, STREAM_1):
self.assertEqual(cmd, vals[0])
self.assertTrue((len(vals) == 1 and not args)
or (vals[1:] == args))
except ProtocolError as e:
print 'error -- %s' % e
def __run_test(self, mode, mapper_class, context_class):
cmd_file = self.__write_cmd_file(mode)
pp.run_task(
pp.Factory(mapper_class=mapper_class), private_encoding=False,
context_class=context_class, cmd_file=cmd_file
)
out_fn = cmd_file + '.out'
out_records = []
with open(out_fn) as ostream:
for cmd, args in BinaryDownStreamFilter(ostream):
if cmd == 'output':
name, color = args
out_records.append({'name': name, 'favorite_color': color})
self.assertEqual(len(out_records), len(self.records))
for out_r, r in zip(out_records, self.records):
for k, v in out_r.iteritems():
self.assertEqual(v, r[k])
def _test_map_reduce_with_private_encoding_helper(self,
factory,
fast_combiner=False):
self.stream3.close()
cmd_file = self.stream3.name
out_file = cmd_file + '.out'
reduce_infile = cmd_file + '.reduce'
reduce_outfile = reduce_infile + '.out'
run_task(factory,
cmd_file=cmd_file,
private_encoding=True,
fast_combiner=fast_combiner)
data = {}
with open(out_file) as f:
bf = BinaryDownStreamFilter(f)
for cmd, args in bf:
if cmd == 'output':
data.setdefault(args[0], []).append(args[1])
stream = []
stream.append(('start', 0))
stream.append(('setJobConf', ('key1', 'value1', 'key2', 'value2')))
stream.append(('runReduce', 0, False))
for k in data:
stream.append(('reduceKey', k))
for v in data[k]:
stream.append(('reduceValue', v))
stream.append(('close', ))
binary_stream_writer(reduce_infile, stream)
run_task(factory, cmd_file=reduce_infile, private_encoding=True)
with open(reduce_outfile) as f, self._mkf('foo.out', mode='w') as o:
bf = BinaryUpStreamDecoder(f)
for cmd, args in bf:
if cmd == 'progress':
o.write('progress\t%s\n' % args[0])
elif cmd == 'output':
o.write('output\t%s\n' % '\t'.join(args))
elif cmd == 'done':
o.write('done\n')
self.check_result('foo.out', STREAM_3)
def decode(istream, ostream):
cmd_stream = BinaryDownStreamFilter(istream)
for (cmd, args) in cmd_stream:
ostream.write('cmd: {}, args: {}\n'.format(cmd, args))
def read_data(fname):
with open(fname, 'rb', buffering=(4096 * 4)) as f:
reader = BinaryDownStreamFilter(f)
for cmd, args in reader:
pass