def get_column_value(self, line): """ 根据行数据获取需要插入到数据库的字典 :param line: :return: """ index = line.find(']') line = (line[index + 1:]).strip() data = self.parser.split_k_v(line) column_value = { 'uid': data['uid'], 'p_type': Product.MINIK.value, 't_type': Terminal.Mobile.value, 'a_type': -1, 'action_time': -1, 'location': -1, 'aimei_object': '', 'update_time': -1 } action = data['action'] if action == 'click_listen_song': column_value['a_type'] = Action.REQUEST_SONG.value column_value['aimei_object'] = data['songid'] else: column_value['a_type'] = Action.CLICK_BUTTON.value aimei_object = '' if action == 'click_tab': aimei_object = data['tab'] elif action == 'visit_url': aimei_object = data['url'] elif action == 'click_play_album': aimei_object = data['albumid'] column_value['aimei_object'] = aimei_object column_value['action_time'] = DateUtil.date2timestamp( data['timestamp'], fmt='%Y-%m-%d_%H:%M:%S') month = DateUtil.now('%Y%m') column_value['month'] = month column_value['update_time'] = DateUtil.now('%Y-%m-%d %H:%M:%S') # 流水需要的 # 总表需要的 column_value['provinceid'] = -1 column_value['isp'] = -1 column_value['count'] = 1 column_value['time_span'] = self.get_time_span(data['timestamp']) return column_value
def process(self, lines): # 格式:[I 160815 00:00:00 record_handler:181] timestamp=2016-08-15_00:00:00&action=click_listen_song&songid=2306797&uid=1277909 column_values = [] total_column_values = [] for line in lines: index = line.find(']') line = (line[index + 1:]).strip() data = self.parser.split_k_v(line) column_value = { 'uid': data['uid'], 'p_type': Product.MINIK.value, 't_type': Terminal.Mobile.value, 'a_type': -1, 'action_time': -1, 'location': -1, 'aimei_object': '', 'update_time': -1 } action = data['action'] if action == 'click_listen_song': column_value['a_type'] = Action.REQUEST_SONG.value column_value['aimei_object'] = data['songid'] else: column_value['a_type'] = Action.CLICK_BUTTON.value aimei_object = '' if action == 'click_tab': aimei_object = data['tab'] elif action == 'visit_url': aimei_object = data['url'] elif action == 'click_play_album': aimei_object = data['albumid'] column_value['aimei_object'] = aimei_object column_value['action_time'] = DateUtil.date2timestamp( data['timestamp'], fmt='%Y-%m-%d_%H:%M:%S') # user_action = UserAction.gen_model(column_value) # user_action.session.add(user_action) # user_action.session.commit() sql, column_value = self.insert(column_value) column_values.append(column_value) sql, total_column_value = self.insert_or_update_total( column_value, data) total_column_values.append(total_column_value) self.insert_many(column_values) self.insert_total_many(total_column_values)