def reducer(self, uid_1, vals_1):
yield "", MRUtils.str_dump({
"uid": uid_1,
"count": len(vals_1)
})
def mapper(self, line1):
item1 = MRUtils.json_parse(line1)
yield item1["uid"], item1
def reducer(self, uid_1, vals_1):
yield "", MRUtils.str_dump(
{"uid": uid_1, "count": len(vals_1)})
def test_mr(self):
from luiti import MRUtils
item1 = {"class_id": 3, "uid": 7}
self.assertEqual(MRUtils.mr_key(item1), "3@@7")
self.assertEqual(MRUtils.mr_key(item1, "hid009"), "3@@7@@hid009")
self.assertEqual(MRUtils.json_parse("{\"你好\":\"世界\"}"), {u"你好": u"世界"})
self.assertFalse(MRUtils.is_mr_line("[1,2]"))
self.assertTrue(MRUtils.is_mr_line("hello\t{framework:luigi}"))
self.assertTrue(MRUtils.is_mr_line("1@@" + "2" * 40 + "\t[world]"))
self.assertEqual(
MRUtils.unicode_value({u"hello": u"世界"}, "hello"), u"世界")
self.assertEqual(
MRUtils.split_mr_kv("hello\t[1,2,3,4]"), ["hello", [1, 2, 3, 4]])
self.assertEqual(
MRUtils.merge_keys_in_dict([{"a": 1}, {"a": 2}], ["a"]), {"a": 3})
self.assertEqual(
MRUtils.split_prefix_keys("1@@2@@other"), ["1", "2", "other"])
prefix_str1 = "232@@8923802@@afenti"
prefix_str2 = "\"" + prefix_str1
self.assertEqual(
MRUtils.select_prefix_keys(prefix_str1, [0, 1]), "232@@8923802")
self.assertEqual(
MRUtils.select_prefix_keys(prefix_str2, [0, 1]), "232@@8923802")
self.assertEqual(
MRUtils.str_dump({"hello": u"世界"}), """{"hello": "世界"}""")
self.assertEqual(
MRUtils.filter_dict(
{"hello": "world", "foobar": "barfoo"}, "hello"),
{"hello": "world"})
def mapper(self, line1):
item1 = MRUtils.json_parse(line1)
yield item1["uid"], item1