def reducer(self, uid_1, vals_1):
     yield "", MRUtils.str_dump(
         {"uid": uid_1, "count": len(vals_1)})
Example #2
0
    def test_mr(self):
        from luiti import MRUtils

        item1 = {"class_id": 3, "uid": 7}
        self.assertEqual(MRUtils.mr_key(item1), "[email protected]@7")
        self.assertEqual(MRUtils.mr_key(item1, "hid009"), "[email protected]@[email protected]@hid009")

        self.assertEqual(MRUtils.json_parse("{\"你好\":\"世界\"}"), {u"你好": u"世界"})

        self.assertFalse(MRUtils.is_mr_line("[1,2]"))
        self.assertTrue(MRUtils.is_mr_line("hello\t{framework:luigi}"))
        self.assertTrue(MRUtils.is_mr_line("[email protected]@" + "2" * 40 + "\t[world]"))

        self.assertEqual(
            MRUtils.unicode_value({u"hello": u"世界"}, "hello"), u"世界")

        self.assertEqual(
            MRUtils.split_mr_kv("hello\t[1,2,3,4]"), ["hello", [1, 2, 3, 4]])

        self.assertEqual(
            MRUtils.merge_keys_in_dict([{"a": 1}, {"a": 2}], ["a"]), {"a": 3})

        self.assertEqual(
            MRUtils.split_prefix_keys("[email protected]@[email protected]@other"), ["1", "2", "other"])

        prefix_str1 = "[email protected]@[email protected]@afenti"
        prefix_str2 = "\"" + prefix_str1
        self.assertEqual(
            MRUtils.select_prefix_keys(prefix_str1, [0, 1]), "[email protected]@8923802")
        self.assertEqual(
            MRUtils.select_prefix_keys(prefix_str2, [0, 1]), "[email protected]@8923802")

        self.assertEqual(
            MRUtils.str_dump({"hello": u"世界"}), """{"hello": "世界"}""")

        self.assertEqual(
            MRUtils.filter_dict(
                {"hello": "world", "foobar": "barfoo"}, "hello"),
            {"hello": "world"})
 def mapper(self, line1):
     item1 = MRUtils.json_parse(line1)
     yield item1["uid"], item1