def parse_output(self, protocol=None):
""".. deprecated:: 0.4.2
Parse the output from the given sandboxed job's ``self.stdout``.
This was only useful for testing individual mappers/reducers
without using a runner; normally you'd just use
:py:meth:`runner.stream_output()
<mrjob.runner.MRJobRunner.stream_output()>`
:type protocol: protocol
:param protocol: A protocol instance to use. Defaults to
``JSONProtocol()``.
"""
if self.stdout == sys.stdout:
raise AssertionError('You must call sandbox() first;'
' parse_output() is for testing only.')
log.warning(
'parse_output() is deprecated and will be removed in v0.5.0')
if protocol is None:
protocol = JSONProtocol()
lines = StringIO(self.stdout.getvalue())
return [protocol.read(line) for line in lines]
def test_uses_json_format(self):
KEY = ['a', 1]
VALUE = {'foo': {'bar': 3}, 'baz': None}
ENCODED = '["a", 1]\t{"foo": {"bar": 3}, "baz": null}'
self.assertEqual((KEY, VALUE), JSONProtocol.read(ENCODED))
self.assertEqual(ENCODED, JSONProtocol.write(KEY, VALUE))
def test_uses_json_format(self):
KEY = ['a', 1]
VALUE = {'foo': {'bar': 3}, 'baz': None}
ENCODED = '["a", 1]\t{"foo": {"bar": 3}, "baz": null}'
self.assertEqual((KEY, VALUE), JSONProtocol.read(ENCODED))
self.assertEqual(ENCODED, JSONProtocol.write(KEY, VALUE))
def test_numerical_keys_become_strs(self):
# JSON should convert numbers to strings when they are dict keys
self.assertEqual(({
'1': 2
}, {
'3': 4
}), JSONProtocol.read(JSONProtocol.write({1: 2}, {3: 4})))
def test_decode(self):
linRegFactory = LinearRegressionFactory(11)
linReg = linRegFactory.get_instance()
obj_encoded = linRegFactory.encode(linReg)
protocol = JSONProtocol()
json_encoded = protocol.write(0, obj_encoded)
obj_encoded = protocol.read(json_encoded)
linRegArr = linRegFactory.decode([obj_encoded[1]])
assert type(linRegArr) == list, "decoded not as a list"
assert type(linRegArr[0]) == LinearRegression, "decoded not as LinearRegression"
def test_decode(self):
linRegFactory = LinearRegressionFactory(11)
linReg = linRegFactory.get_instance()
obj_encoded = linRegFactory.encode(linReg)
protocol = JSONProtocol()
json_encoded = protocol.write(0, obj_encoded)
obj_encoded = protocol.read(json_encoded)
linRegArr = linRegFactory.decode([obj_encoded[1]])
assert type(linRegArr) == list, "decoded not as a list"
assert type(linRegArr[0]
) == LinearRegression, "decoded not as LinearRegression"
def test_decode(self):
'''
Test whether algorithm can be json encoded (used as mrjob internal protocol)
'''
layerSizes = [3,2,1]
nnFactory = PredictionNNFactory(layerSizes)
nn = nnFactory.get_instance()
# encode
obj_encoded = nnFactory.encode(nn)
# call json protocol
protocol = JSONProtocol()
json_encoded = protocol.write("test_decode", obj_encoded)
obj_encoded = protocol.read(json_encoded)
nnArr = nnFactory.decode([obj_encoded[1]])
assert type(nnArr) == list, "decoded not as a list"
assert type(nnArr[0]) == MultilayerPerceptron, "decoded not as LinearRegression"
def test_numerical_keys_become_strs(self):
# JSON should convert numbers to strings when they are dict keys
self.assertEqual(({'1': 2}, {'3': 4}),
JSONProtocol.read(JSONProtocol.write({1: 2}, {3: 4})))
def test_tuples_become_lists(self):
# JSON should convert tuples into lists
self.assertEqual(([1, 2], [3, 4]),
JSONProtocol.read(JSONProtocol.write((1, 2), (3, 4))))
def test_tuples_become_lists(self):
# JSON should convert tuples into lists
self.assertEqual(([1, 2], [3, 4]),
JSONProtocol.read(JSONProtocol.write((1, 2), (3, 4))))
def read(string):
list_, prob = JSONProtocol.read(string)
return tuple(list_), prob
