def _get_reader(self, f):
# ctypedbytes segfaults on these pseudo-file-likes
inp = typedbytes.Input(f)
register_read(inp)
for args, kwargs in self.registrations:
inp.register(*args, **kwargs)
return inp
def testpickle(self):
file = open("test.bin", "wb")
output = typedbytes.Output(file)
output.write("123")
output.write(MyClass())
file.close()
file = open("test.bin", "rb")
input = typedbytes.Input(file)
self.assertEqual("123", input.read())
self.assertEqual(234, input.read().attr)
file.close()
os.remove("test.bin")
def testio(self):
objects = TestIO.objects
file = open("test.bin", "wb")
output = typedbytes.Output(file)
output.writes(objects)
file.close()
file = open("test.bin", "rb")
input = typedbytes.Input(file)
for (index, record) in enumerate(input):
self.assertEqual(objects[index], record)
file.close()
os.remove("test.bin")
def testints(self):
file = cStringIO.StringIO()
output = typedbytes.Output(file)
t = time.time()
output.writes(xrange(100000))
print time.time() - t
file.seek(0)
input = typedbytes.Input(file)
t = time.time()
for record in input:
pass
print time.time() - t
file.close()
import numpy as np
import typedbytes as tb
import sys
num_reducers = int(sys.argv[2])
tau_vec = []
s_vec = []
values = []
for i in range(num_reducers):
fn = sys.argv[1] + '/X_000%02d.tb' %i
reader = tb.Input(open(fn, 'rb')).reads()
for key in reader:
tau_vec += [0.5, 0.75, 0.95]
s_vec.append(key[1])
values += reader.next()
X = np.array(values).T
m, n = X.shape
X = np.vstack([X, -np.ones([1, n])])
np.savetxt(sys.argv[1]+'/sv.txt', s_vec)
np.savetxt(sys.argv[1]+'/tv.txt', tau_vec)
np.savetxt(sys.argv[1]+'/X.txt', X)