コード例 #1
0
def fun_map_json(json_string, time_key, time_interval_minute_list, time_key_prefix):
    import json
    reload(sys)
    sys.setdefaultencoding("utf-8")

    from datetime import datetime

    obj = json.loads(json_string)

    time = obj.get(time_key, '')
    # print('= = ' * 10, '[myapp test_utils.fun_map_row] found time = ', time, ', obj = ', obj)
    cst_time = date_string_utc2cst(time)
    obj[time_key] = cst_time
    if cst_time == '':
        res = None
        print('= = ' * 10, '[myapp test_utils.fun_map_row] found ivalid cst_time')
    else:
        labels = obj['labels']
        minute = datetime.strptime(cst_time, '%Y-%m-%d %H:%M:%S').minute

        for time_interval_minute in time_interval_minute_list:
            minute_start = (minute / time_interval_minute) * time_interval_minute
            minute_end = minute_start + time_interval_minute - 1
            time_start_key = time_key_prefix + str(time_interval_minute) + '_start'
            time_end_key = time_key_prefix + str(time_interval_minute) + '_end'
            time_start = cst_time[0:14] + str(minute_start).zfill(2) + ':00'
            time_end = cst_time[0:14] + str(minute_end).zfill(2) + ':59'
            labels[time_start_key] = time_start
            labels[time_end_key] = time_end
        obj['labels'] = labels

        res = Row(**obj)
        print('= = ' * 10, '[myapp test_utils.fun_map_row] res.time = ', res.time, ', labels = ', res.labels)

    return res
コード例 #2
0
    def fun_time_in_rdd_map(row,
                            time_key,
                            time_interval_minute_list,
                            time_key_prefix,
                            cache_conf=None):
        """
        输入: Row[**obj]
        输出: Row[**obj]
        """
        import sys

        reload(sys)
        sys.setdefaultencoding("utf-8")

        from datetime import datetime
        # obj = json.loads(line)
        obj = row.asDict()

        time = obj.get(time_key, '')
        cst_time = date_string_utc2cst(time)
        if cst_time == '':
            ret = None
        else:
            obj[time_key] = cst_time
            minute = datetime.strptime(cst_time, '%Y-%m-%d %H:%M:%S').minute

            for time_interval_minute in time_interval_minute_list:
                minute_start = (minute /
                                time_interval_minute) * time_interval_minute
                minute_end = minute_start + time_interval_minute - 1
                time_start_key = time_key_prefix + str(
                    time_interval_minute) + '_start'
                time_end_key = time_key_prefix + str(
                    time_interval_minute) + '_end'
                time_start = cst_time[0:14] + str(minute_start).zfill(
                    2) + ':00'
                time_end = cst_time[0:14] + str(minute_end).zfill(2) + ':59'
                obj[time_start_key] = time_start
                obj[time_end_key] = time_end

            # ret = json.dumps(obj)
            ret = Row(**obj)
        # print('= = ' * 15, '[myapp EnhanceTimeProcessor.process.fun_time_in_rdd_map] ret = ')
        # print(ret)
        return ret
コード例 #3
0
def fun_map_json(json_string, time_key, time_interval_minute_list,
                 time_key_prefix):
    import json
    reload(sys)
    sys.setdefaultencoding("utf-8")

    from datetime import datetime

    obj = json.loads(json_string)

    time = obj.get(time_key, '')
    # print('= = ' * 10, '[myapp test_utils.fun_map_row] found time = ', time, ', obj = ', obj)
    cst_time = date_string_utc2cst(time)
    obj[time_key] = cst_time
    if cst_time == '':
        res = None
        print('= = ' * 10,
              '[myapp test_utils.fun_map_row] found ivalid cst_time')
    else:
        labels = obj['labels']
        minute = datetime.strptime(cst_time, '%Y-%m-%d %H:%M:%S').minute

        for time_interval_minute in time_interval_minute_list:
            minute_start = (minute /
                            time_interval_minute) * time_interval_minute
            minute_end = minute_start + time_interval_minute - 1
            time_start_key = time_key_prefix + str(
                time_interval_minute) + '_start'
            time_end_key = time_key_prefix + str(time_interval_minute) + '_end'
            time_start = cst_time[0:14] + str(minute_start).zfill(2) + ':00'
            time_end = cst_time[0:14] + str(minute_end).zfill(2) + ':59'
            labels[time_start_key] = time_start
            labels[time_end_key] = time_end
        obj['labels'] = labels

        res = Row(**obj)
        print('= = ' * 10, '[myapp test_utils.fun_map_row] res.time = ',
              res.time, ', labels = ', res.labels)

    return res
コード例 #4
0
    def fun_time_in_rdd_map(row, time_key, time_interval_minute_list, time_key_prefix, cache_conf=None):
        """
        输入: Row[**obj]
        输出: Row[**obj]
        """
        import sys

        reload(sys)
        sys.setdefaultencoding("utf-8")

        from datetime import datetime
        # obj = json.loads(line)
        obj = row.asDict()

        time = obj.get(time_key, '')
        cst_time = date_string_utc2cst(time)
        if cst_time == '':
            ret = None
        else:
            obj[time_key] = cst_time
            minute = datetime.strptime(cst_time, '%Y-%m-%d %H:%M:%S').minute

            for time_interval_minute in time_interval_minute_list:
                minute_start = (minute / time_interval_minute) * time_interval_minute
                minute_end = minute_start + time_interval_minute - 1
                time_start_key = time_key_prefix + str(time_interval_minute) + '_start'
                time_end_key = time_key_prefix + str(time_interval_minute) + '_end'
                time_start = cst_time[0:14] + str(minute_start).zfill(2) + ':00'
                time_end = cst_time[0:14] + str(minute_end).zfill(2) + ':59'
                obj[time_start_key] = time_start
                obj[time_end_key] = time_end

            # ret = json.dumps(obj)
            ret = Row(**obj)
        # print('= = ' * 15, '[myapp EnhanceTimeProcessor.process.fun_time_in_rdd_map] ret = ')
        # print(ret)
        return ret