Exemple #1
0
    def test_one_level_deep_flattens(self):
        data = dict(flat1=1, dict1=dict(c=1, d=2))

        result = nested_to_record(data)
        expected = {'dict1.c': 1, 'dict1.d': 2, 'flat1': 1}

        self.assertEqual(result, expected)
Exemple #2
0
    def test_flat_stays_flat(self):
        recs = [dict(flat1=1, flat2=2),
                dict(flat1=3, flat2=4),
                ]

        result = nested_to_record(recs)
        expected = recs
        self.assertEqual(result, expected)
Exemple #3
0
    def test_flat_stays_flat(self):
        recs = [dict(flat1=1,flat2=2),
                dict(flat1=3,flat2=4),
                ]

        result = nested_to_record(recs)
        expected = recs
        self.assertEqual(result, expected)
Exemple #4
0
 def json_to_norm_dict(s, c):
     logging.info(str(c))
     normalized_dict = pjson.nested_to_record(json.loads(s))
     return {
         key.replace('.', '_'): value
         for key, value in normalized_dict.iteritems()
         if key in json_data_columns.WANTED_COLUMNS_STOCKTWITS
     }
Exemple #5
0
    def test_one_level_deep_flattens(self):
        data = dict(flat1=1,
                    dict1=dict(c=1,d=2))

        result = nested_to_record(data)
        expected =     {'dict1.c': 1,
             'dict1.d': 2,
             'flat1': 1}

        self.assertEqual(result,expected)
Exemple #6
0
    def test_nested_flattens(self):
        data = dict(flat1=1,
                    dict1=dict(c=1, d=2),
                    nested=dict(e=dict(c=1, d=2),
                                d=2))

        result = nested_to_record(data)
        expected = {'dict1.c': 1,
                    'dict1.d': 2,
                    'flat1': 1,
                    'nested.d': 2,
                    'nested.e.c': 1,
                    'nested.e.d': 2}

        self.assertEqual(result, expected)
Exemple #7
0
    def test_nested_flattens(self):
        data = dict(flat1=1,
                    dict1=dict(c=1,d=2),
                    nested=dict(e=dict(c=1,d=2),
                                d=2))

        result = nested_to_record(data)
        expected =     {'dict1.c': 1,
             'dict1.d': 2,
             'flat1': 1,
             'nested.d': 2,
             'nested.e.c': 1,
             'nested.e.d': 2}

        self.assertEqual(result,expected)
Exemple #8
0
def read_nested_write_flat(infile, outfile):
    with open(infile, 'r') as infile:
        with open(outfile, 'a') as outfile:
            for record in read_jsonl_stream(infile):
                dump_jsonl_stream(nested_to_record(record), outfile)
    print "saved records to {}".format(outfile.name)