def parse_output(self, protocol=None): """.. deprecated:: 0.4.2 Parse the output from the given sandboxed job's ``self.stdout``. This was only useful for testing individual mappers/reducers without using a runner; normally you'd just use :py:meth:`runner.stream_output() <mrjob.runner.MRJobRunner.stream_output()>` :type protocol: protocol :param protocol: A protocol instance to use. Defaults to ``JSONProtocol()``. """ if self.stdout == sys.stdout: raise AssertionError('You must call sandbox() first;' ' parse_output() is for testing only.') log.warning( 'parse_output() is deprecated and will be removed in v0.5.0') if protocol is None: protocol = JSONProtocol() lines = StringIO(self.stdout.getvalue()) return [protocol.read(line) for line in lines]
def test_uses_json_format(self): KEY = ['a', 1] VALUE = {'foo': {'bar': 3}, 'baz': None} ENCODED = '["a", 1]\t{"foo": {"bar": 3}, "baz": null}' self.assertEqual((KEY, VALUE), JSONProtocol.read(ENCODED)) self.assertEqual(ENCODED, JSONProtocol.write(KEY, VALUE))
def test_numerical_keys_become_strs(self): # JSON should convert numbers to strings when they are dict keys self.assertEqual(({ '1': 2 }, { '3': 4 }), JSONProtocol.read(JSONProtocol.write({1: 2}, {3: 4})))
def test_decode(self): linRegFactory = LinearRegressionFactory(11) linReg = linRegFactory.get_instance() obj_encoded = linRegFactory.encode(linReg) protocol = JSONProtocol() json_encoded = protocol.write(0, obj_encoded) obj_encoded = protocol.read(json_encoded) linRegArr = linRegFactory.decode([obj_encoded[1]]) assert type(linRegArr) == list, "decoded not as a list" assert type(linRegArr[0]) == LinearRegression, "decoded not as LinearRegression"
def test_decode(self): linRegFactory = LinearRegressionFactory(11) linReg = linRegFactory.get_instance() obj_encoded = linRegFactory.encode(linReg) protocol = JSONProtocol() json_encoded = protocol.write(0, obj_encoded) obj_encoded = protocol.read(json_encoded) linRegArr = linRegFactory.decode([obj_encoded[1]]) assert type(linRegArr) == list, "decoded not as a list" assert type(linRegArr[0] ) == LinearRegression, "decoded not as LinearRegression"
def test_decode(self): ''' Test whether algorithm can be json encoded (used as mrjob internal protocol) ''' layerSizes = [3,2,1] nnFactory = PredictionNNFactory(layerSizes) nn = nnFactory.get_instance() # encode obj_encoded = nnFactory.encode(nn) # call json protocol protocol = JSONProtocol() json_encoded = protocol.write("test_decode", obj_encoded) obj_encoded = protocol.read(json_encoded) nnArr = nnFactory.decode([obj_encoded[1]]) assert type(nnArr) == list, "decoded not as a list" assert type(nnArr[0]) == MultilayerPerceptron, "decoded not as LinearRegression"
def test_numerical_keys_become_strs(self): # JSON should convert numbers to strings when they are dict keys self.assertEqual(({'1': 2}, {'3': 4}), JSONProtocol.read(JSONProtocol.write({1: 2}, {3: 4})))
def test_tuples_become_lists(self): # JSON should convert tuples into lists self.assertEqual(([1, 2], [3, 4]), JSONProtocol.read(JSONProtocol.write((1, 2), (3, 4))))
def read(string): list_, prob = JSONProtocol.read(string) return tuple(list_), prob