def __init__(self, device_audio_record): self.device_audio_record = device_audio_record config_obj = ConfigReader() config_key = "cassandra" cassandra_config = config_obj.get_section_config(config_key) cassandra_host = [ host.strip() for host in cassandra_config.get("hosts").split(',') ] cassandra_user = cassandra_config.get("username") cassandra_password = cassandra_config.get("password") self.cassandra_connector = CassandraConnector() self.cassandra_connector.db = cassandra_host self.cassandra_connector.username = cassandra_user self.cassandra_connector.password = cassandra_password
def __init__(self, status_value, delay_value, data_record, stop_flag): self.status_value = status_value self.status_value.set(1) self.delay_value = delay_value self.data_record = data_record self.stop_flag = stop_flag config_obj = ConfigReader() config_key = "audio_queue" audio_queue_config = config_obj.get_section_config(config_key) self.queue_ip = audio_queue_config.get("host") self.queue_port = int(audio_queue_config.get("port")) self.queue_key = audio_queue_config.get("key").encode() config_key = "cassandra" cassandra_config = config_obj.get_section_config(config_key) cassandra_host = [ host.strip() for host in cassandra_config.get("hosts").split(',') ] cassandra_user = cassandra_config.get("username") cassandra_password = cassandra_config.get("password") self.cassandra_connector = CassandraConnector() self.cassandra_connector.db = cassandra_host self.cassandra_connector.username = cassandra_user self.cassandra_connector.password = cassandra_password self.audio_queue = self.get_audio_queue() if self.audio_queue is False: logging.error("get audio queue error") sys.exit(1) self.period_logs_info = {} self.device_audio_record = Manager().dict()
class CalculateOneMinuteLogs(object): def __init__(self, device_audio_record): self.device_audio_record = device_audio_record config_obj = ConfigReader() config_key = "cassandra" cassandra_config = config_obj.get_section_config(config_key) cassandra_host = [ host.strip() for host in cassandra_config.get("hosts").split(',') ] cassandra_user = cassandra_config.get("username") cassandra_password = cassandra_config.get("password") self.cassandra_connector = CassandraConnector() self.cassandra_connector.db = cassandra_host self.cassandra_connector.username = cassandra_user self.cassandra_connector.password = cassandra_password def init_cassandra(self): if not self.cassandra_connector.prepare_stmt( "update_device_audio_data", "data_dictionary", "update device_audio_daily set start_type=?,end_type=?,end_time=?,live_status=?,pid=?,tid=?,netstat=? where date=? and device_id=? and start_time=? and phid=?" ): logging.error("init cassandra db error") sys.exit(1) def run(self, calculate_logs): self.init_cassandra() result = self.calculate_audio_logs(calculate_logs) jobs = {} job_index = 0 if result is not None: for device_id in result: for phid in result[device_id]: for start_time in result[device_id][phid]: end_type = result[device_id][phid][start_time][ "end_type"] end_time = result[device_id][phid][start_time][ "end_time"] start_type = result[device_id][phid][start_time][ "start_type"] live_status = result[device_id][phid][start_time][ "live_status"] pid = result[device_id][phid][start_time]["pid"] tid = result[device_id][phid][start_time]["tid"] netstat = None if end_time is not None and end_time < start_time: end_time = start_time + timedelta(seconds=30) jobs[job_index] = [ start_type, end_type, convert_into_utc_time(end_time) if end_time is not None else None, live_status, pid, tid, netstat, convert_into_utc_time( start_time.replace(second=0, minute=0, hour=0)), device_id, convert_into_utc_time(start_time), phid ] job_index += 1 if len(jobs) > 0: try: self.cassandra_connector.do_futures("update_device_audio_data", jobs) except: error_data = traceback.format_exc() logging.error( "update device audio data error: {0}".format(error_data)) def calculate_audio_logs(self, audio_logs): devices_logs = self.get_devices_logs(audio_logs) if len(devices_logs) == 0: return None devices_audio_info = self.get_devices_audio_info(devices_logs) return devices_audio_info def get_devices_audio_info(self, devices_logs): devices_audio_info = {} for device_id in devices_logs: device_logs = self.adjust_logs(devices_logs[device_id]) for log_info in device_logs: action = log_info.get("action") ts = log_info.get("ts") phid = log_info.get("phid") if action is None or phid is None: continue try: phid = int(phid) except: logging.error("the wrong phid: {0}".format(phid)) continue try: pid = int(log_info.get("pid")) except: # logging.warning("the wrong pid: {0}".format(log_info.get("pid"))) pid = 0 try: tid = int(log_info.get("tid")) except: # logging.warning("the wrong tid: {0}".format(log_info.get("tid"))) tid = 0 try: live = int(log_info.get("live")) except: # logging.warning("the wroing live status: {0}".format(log_info.get("live"))) live = 0 if live < 0: live = 0 adjust_log_info = { "phid": phid, "pid": pid, "tid": tid, "live": live } if device_id not in self.device_audio_record: if action not in ["stop", "pause", "complete"]: start_type = "start" self.device_audio_record[device_id] = { "action": action, "start_type": start_type, "start_ts": ts, "update_ts": ts, "phid": phid, "pid": pid, "tid": tid, "live": None if action in ["start", "buffer"] else live } if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if phid not in devices_audio_info[device_id]: devices_audio_info[device_id][phid] = {} devices_audio_info[device_id][phid][ts] = { "start_type": start_type, "end_type": None, "end_time": None, "live_status": None if action in ["start", "buffer"] else live, "pid": pid, "tid": tid } else: device_last_log = self.device_audio_record[device_id] last_action = device_last_log.get("action") last_update_ts = device_last_log.get("update_ts") last_start_type = device_last_log.get("start_type") last_start_ts = device_last_log.get("start_ts") last_phid = device_last_log.get("phid") last_live = device_last_log.get("live") last_pid = device_last_log.get("pid") last_tid = device_last_log.get("tid") if action in ["start", "stop", "complete"]: if last_action == "pause": last_end_ts = last_update_ts last_end_type = "pause" else: if action == "start": diff_ts = int(ts.timestamp() - last_update_ts.timestamp()) if diff_ts >= 40: last_end_ts = last_update_ts + timedelta( seconds=30) else: last_end_ts = ts last_end_type = "stop" else: last_end_ts = ts if action == "complete": last_end_type = "complete" else: last_end_type = "stop" if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if last_phid not in devices_audio_info[device_id]: devices_audio_info[device_id][last_phid] = {} devices_audio_info[device_id][last_phid][ last_start_ts] = { "start_type": last_start_type, "end_type": last_end_type, "end_time": last_end_ts, "live_status": last_live, "pid": last_pid, "tid": last_tid } del self.device_audio_record[device_id] if action == "start": self.device_audio_record[device_id] = { "action": action, "start_type": "start", "start_ts": ts, "update_ts": ts, "phid": phid, "pid": pid, "tid": tid, "live": None if action in ["start", "buffer"] else live } if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if phid not in devices_audio_info[device_id]: devices_audio_info[device_id][phid] = {} devices_audio_info[device_id][phid][ts] = { "start_type": "start", "end_type": None, "end_time": None, "live_status": None if action in ["start", "buffer"] else live, "pid": pid, "tid": tid } elif action == "pause": last_end_ts = ts last_end_type = "pause" self.device_audio_record[device_id] = { "action": action, "start_type": last_start_type, "start_ts": last_start_ts, "update_ts": ts, "phid": last_phid, "pid": last_pid, "tid": last_tid, "live": live } if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if last_phid not in devices_audio_info[device_id]: devices_audio_info[device_id][last_phid] = {} devices_audio_info[device_id][last_phid][ last_start_ts] = { "start_type": last_start_type, "end_type": last_end_type, "end_time": last_end_ts, "live_status": live, "pid": last_pid, "tid": last_tid } else: if not self.is_same_target(adjust_log_info, device_last_log): if action == "buffer": pass else: if last_action == "pause": last_end_ts = last_update_ts last_end_type = "pause" else: diff_ts = int( (ts - last_update_ts).total_seconds()) if diff_ts >= 40: last_end_ts = last_update_ts + timedelta( seconds=30) else: last_end_ts = ts last_end_type = "stop" if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if last_phid not in devices_audio_info[ device_id]: devices_audio_info[device_id][ last_phid] = {} devices_audio_info[device_id][last_phid][ last_start_ts] = { "start_type": last_start_type, "end_type": last_end_type, "end_time": last_end_ts, "live_status": last_live, "pid": last_pid, "tid": last_tid } self.device_audio_record[device_id] = { "action": action, "start_type": "start", "start_ts": ts, "update_ts": ts, "phid": phid, "pid": pid, "tid": tid, "live": live } if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if phid not in devices_audio_info[device_id]: devices_audio_info[device_id][phid] = {} devices_audio_info[device_id][phid][ts] = { "start_type": "start", "end_type": None, "end_time": None, "live_status": live, "pid": pid, "tid": tid } else: if action == "buffer": self.device_audio_record[device_id] = { "action": action, "start_type": last_start_type, "start_ts": last_start_ts, "update_ts": ts, "phid": last_phid, "pid": last_pid, "tid": last_tid, "live": last_live } else: diff_ts = int( (ts - last_update_ts).total_seconds()) if diff_ts >= 40: if last_action == "pause": last_end_ts = last_update_ts last_end_type = "pause" else: last_end_ts = last_update_ts + timedelta( seconds=30) last_end_type = "stop" if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if last_phid not in devices_audio_info[ device_id]: devices_audio_info[device_id][ last_phid] = {} devices_audio_info[device_id][last_phid][ last_start_ts] = { "start_type": last_start_type, "end_type": last_end_type, "end_time": last_end_ts, "live_status": last_live, "pid": last_pid, "tid": last_tid } self.device_audio_record[device_id] = { "action": action, "start_type": "start", "start_ts": ts, "update_ts": ts, "phid": phid, "pid": pid, "tid": tid, "live": live } if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if phid not in devices_audio_info[ device_id]: devices_audio_info[device_id][ phid] = {} devices_audio_info[device_id][phid][ts] = { "start_type": "start", "end_type": None, "end_time": None, "live_status": live, "pid": pid, "tid": tid } else: if last_action == "pause": start_type = "resume" self.device_audio_record[device_id] = { "action": action, "start_type": start_type, "start_ts": ts, "update_ts": ts, "phid": phid, "pid": pid, "tid": tid, "live": live } if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if phid not in devices_audio_info[ device_id]: devices_audio_info[device_id][ phid] = {} devices_audio_info[device_id][phid][ ts] = { "start_type": start_type, "end_type": None, "end_time": None, "live_status": live, "pid": pid, "tid": tid } else: self.device_audio_record[device_id] = { "action": action, "start_type": last_start_type, "start_ts": last_start_ts, "update_ts": ts, "phid": last_phid, "pid": last_pid, "tid": last_tid, "live": live } if device_id in devices_audio_info: if last_phid in devices_audio_info[ device_id]: if last_start_ts in devices_audio_info[ device_id][last_phid]: if devices_audio_info[ device_id][last_phid][ last_start_ts][ "live_status"] is None: devices_audio_info[ device_id][last_phid][ last_start_ts][ "live_status"] = live return devices_audio_info def is_same_target(self, a, b): for k in ["phid", "pid", "tid", "live"]: if k == "live" and b.get(k) is None: continue if a.get(k) != b.get(k): return False return True def adjust_logs(self, lines): lines = [self.radio_extract(line) for line in lines] for line in lines: if line.get("client_ts") is None: line["client_ts"] = line.get("ts") min_server_ts = min([l.get("ts") for l in lines]) lines = sorted(lines, key=lambda l: self.action_sort(l.get("action"))) lines = sorted(lines, key=lambda l: l.get("client_ts")) min_client_ts = min([l.get("client_ts") for l in lines]) for line in lines: line["server_ts"] = line.get("ts") line["ts"] = min_server_ts + (line.get("client_ts") - min_client_ts) return lines def action_sort(self, action): weight = {"start": 1, "stop": 0} return 2 if weight.get(action) is None else weight.get(action) def radio_extract(self, line): params = line["params_info"] ret = {"action": params.get("t2"), "ts": line.get("log_time")} for k in ["pid", "phid", "live", "tid"]: ret[k] = params.get(k) if params.get("ts") is None: ret["client_ts"] = line.get("log_time") elif len(str(params.get("ts"))) >= 10: try: if len(str(params.get("ts"))) > 10: ts = int(params.get("ts")) / 1000 else: ts = int(params.get("ts")) # ts = int(params.get("ts")) / (10 ** (len(str(params.get("ts"))) - 10)) ret["client_ts"] = datetime.fromtimestamp(ts) except: logging.error("got client ts {0}".format(params.get("ts"))) ret["client_ts"] = line.get("log_time") else: ret["client_ts"] = line.get("log_time") return ret def get_devices_last_log(self, devices_logs): devices_last_log = {} for device in devices_logs: device_logs = sorted(devices_logs[device], key=lambda l: l["log_time"], reverse=True) log_info = device_logs[0] params_info = log_info.get("params_info") action = params_info.get("t2") phid = params_info.get("phid") if action is None or phid is None: continue phid = int(phid) devices_last_log[device] = { "action": action, "ts": log_info.get("log_time"), "phid": phid } for key in ["pid", "tid", "live"]: devices_last_log[device][key] = int(params_info.get( key)) if params_info.get(key) is not None else 0 return devices_last_log def get_devices_logs(self, logs_info): devices_logs = {} for log_info in logs_info: device = log_info.get("device") if device not in devices_logs: devices_logs[device] = [] devices_logs[device].append(log_info) return devices_logs
class CalculateOneMinuteLogs(object): def __init__(self, device_audio_record): self.device_audio_record = device_audio_record config_obj = ConfigReader() config_key = "cassandra" cassandra_config = config_obj.get_section_config(config_key) cassandra_host = [ host.strip() for host in cassandra_config.get("hosts").split(',') ] cassandra_user = cassandra_config.get("username") cassandra_password = cassandra_config.get("password") self.cassandra_connector = CassandraConnector() self.cassandra_connector.db = cassandra_host self.cassandra_connector.username = cassandra_user self.cassandra_connector.password = cassandra_password def init_cassandra(self): if not self.cassandra_connector.prepare_stmt( "update_device_audio_data", "data_dictionary", "update device_audio_daily set start_type=?,end_type=?,end_time=?,live_status=?,pid=?,tid=?,netstat=?,start_pos=?,end_pos=?,album_id=?,controller=?,audio_id=?,key_id=?,user_id=? where date=? and device_id=? and start_time=? and phid=?" ): logging.error("init cassandra db error") sys.exit(1) def run(self, calculate_logs): self.init_cassandra() self.action_adjust_list = [("stop", "start"), ("complete", "start"), ("start", "play"), ("play", "complete"), ("play", "stop"), ("play", "pause"), ("resume", "play"), ("pause", "resume"), ("seek_start", "start")] result = self.calculate_audio_logs(calculate_logs) jobs = {} job_index = 0 if result is not None: for device_id in result: for phid in result[device_id]: for start_time in result[device_id][phid]: end_type = result[device_id][phid][start_time][ "end_type"] end_time = result[device_id][phid][start_time][ "end_time"] start_type = result[device_id][phid][start_time][ "start_type"] live_status = result[device_id][phid][start_time][ "live_status"] pid = result[device_id][phid][start_time]["pid"] tid = result[device_id][phid][start_time]["tid"] album_id = result[device_id][phid][start_time][ "album_id"] netstat = result[device_id][phid][start_time][ "netstat"] controller = result[device_id][phid][start_time][ "controller"] start_pos = result[device_id][phid][start_time][ "start_pos"] end_pos = result[device_id][phid][start_time][ "end_pos"] audio_id = result[device_id][phid][start_time][ "audio_id"] key_id = result[device_id][phid][start_time]["key_id"] user_id = result[device_id][phid][start_time][ "user_id"] if end_time is not None and end_time < start_time: end_time = start_time + timedelta(seconds=30) jobs[job_index] = [ start_type, end_type, convert_into_utc_time(end_time) if end_time is not None else None, live_status, pid, tid, netstat, start_pos, end_pos, album_id, controller, audio_id, key_id, user_id, convert_into_utc_time( start_time.replace(second=0, minute=0, hour=0)), device_id, convert_into_utc_time(start_time), phid ] job_index += 1 if len(jobs) > 0: try: self.cassandra_connector.do_futures("update_device_audio_data", jobs) except: error_data = traceback.format_exc() logging.error( "update device audio data error: {0}".format(error_data)) def calculate_audio_logs(self, audio_logs): devices_logs = self.get_devices_logs(audio_logs) if len(devices_logs) == 0: return None devices_audio_info = self.get_devices_audio_info(devices_logs) return devices_audio_info def get_devices_audio_info(self, devices_logs): devices_audio_info = {} for device_id in devices_logs: device_logs = self.adjust_logs(devices_logs[device_id]) for log_info in device_logs: action = log_info.get("params_info").get("t2") ts = log_info.get("log_time") phid = log_info.get("params_info").get("phid") if action is None or phid is None: continue try: phid = int(phid) except: logging.error("the wrong phid: {0}".format(phid)) continue action = action.lower() try: pid = int(log_info.get("params_info").get("pid")) except: # logging.warning("the wrong pid: {0}".format(log_info.get("params_info").get("pid"))) pid = 0 try: tid = int(log_info.get("params_info").get("tid")) except: # logging.warning("the wrong tid: {0}".format(log_info.get("params_info").get("tid"))) tid = 0 try: live = int(log_info.get("params_info").get("islive")) except: # logging.warning("the wrong live status: {0}".format(log_info.get("params_info").get("islive"))) live = 0 try: album_id = int(log_info.get("params_info").get("albumId")) except: # logging.warning("the wrong album id: {0}".format(log_info.get("params_info").get("albumId"))) album_id = 0 try: audio_id = int(log_info.get("params_info").get("audioId")) except: audio_id = None try: key_id = int(log_info.get("params_info").get("keyId")) except: key_id = None try: user_id = int(log_info.get("params_info").get("userId")) except: user_id = 0 play_time = int( round(float(log_info.get("params_info").get("playtime")), 0)) if log_info.get("params_info").get( "playtime") is not None else None start_time = int( round(float(log_info.get("params_info").get("startTime")), 0)) if log_info.get("params_info").get( "startTime") is not None else None target_time = int( round(float(log_info.get("params_info").get("targetTime")), 0)) if log_info.get("params_info").get( "targetTime") is not None else None total_time = int( round(float(log_info.get("params_info").get("totalTime")), 0)) if log_info.get("params_info").get( "totalTime") is not None else None netstat = log_info.get("params_info").get("netstat") controller = log_info.get("params_info").get("controller") if live < 0: live = 0 adjust_log_info = { "phid": phid, "pid": pid, "user_id": user_id, "tid": tid, "live": live, "album_id": album_id } if device_id not in self.device_audio_record: if action not in [ "stop", "pause", "complete", "buffer", "buffer_full" ]: if action == "resume": start_type = "resume" start_pos = play_time elif action == "seek_start": start_type = "seek_start" start_pos = target_time elif action == "play": start_type = "play" start_pos = play_time else: start_type = "start" start_pos = None if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if phid not in devices_audio_info[device_id]: devices_audio_info[device_id][phid] = {} devices_audio_info[device_id][phid][ts] = { "start_type": start_type, "end_type": None, "end_time": None, "live_status": None if action in ["start"] else live, "pid": pid, "tid": tid, "album_id": album_id, "start_pos": start_pos, "end_pos": None, "netstat": netstat, "controller": controller, "audio_id": audio_id, "key_id": key_id, "user_id": user_id } self.device_audio_record[device_id] = { "action": action, "start_type": start_type, "start_ts": ts, "update_ts": ts, "phid": phid, "pid": pid, "tid": tid, "album_id": album_id, "audio_id": audio_id, "key_id": key_id, "user_id": user_id, "live": None if action in ["start"] else live, "start_pos": start_pos, "end_pos": None, "netstat": netstat, "controller": controller } else: device_last_log = self.device_audio_record[device_id] last_action = device_last_log.get("action") last_update_ts = device_last_log.get("update_ts") last_start_type = device_last_log.get("start_type") last_start_ts = device_last_log.get("start_ts") last_phid = device_last_log.get("phid") last_live = device_last_log.get("live") last_pid = device_last_log.get("pid") last_tid = device_last_log.get("tid") last_album_id = device_last_log.get("album_id") last_start_pos = device_last_log.get("start_pos") last_end_pos = device_last_log.get("end_pos") last_netstat = device_last_log.get("netstat") last_controller = device_last_log.get("controller") last_audio_id = device_last_log.get("audio_id") last_key_id = device_last_log.get("key_id") last_user_id = device_last_log.get("user_id") if action in [ "start", "seek_start", "pause", "stop", "complete" ]: if action == "start": diff_ts = int(ts.timestamp() - last_update_ts.timestamp()) if diff_ts >= 40: last_end_ts = last_update_ts + timedelta( seconds=30) else: last_end_ts = ts last_end_type = "stop" if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if last_phid not in devices_audio_info[device_id]: devices_audio_info[device_id][last_phid] = {} devices_audio_info[device_id][last_phid][ last_start_ts] = { "start_type": last_start_type, "end_type": last_end_type, "end_time": last_end_ts, "live_status": last_live, "pid": last_pid, "tid": last_tid, "album_id": last_album_id, "start_pos": last_start_pos, "end_pos": last_end_pos, "netstat": last_netstat, "controller": last_controller, "audio_id": last_audio_id, "key_id": last_key_id, "user_id": last_user_id } del self.device_audio_record[device_id] start_type = "start" if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if phid not in devices_audio_info[device_id]: devices_audio_info[device_id][phid] = {} devices_audio_info[device_id][phid][ts] = { "start_type": start_type, "end_type": None, "end_time": None, "live_status": None, "pid": pid, "tid": tid, "album_id": album_id, "start_pos": None, "end_pos": None, "netstat": netstat, "controller": controller, "audio_id": audio_id, "key_id": key_id, "user_id": user_id } self.device_audio_record[device_id] = { "action": action, "start_type": start_type, "start_ts": ts, "update_ts": ts, "phid": phid, "pid": pid, "tid": tid, "album_id": album_id, "live": None, "start_pos": None, "end_pos": None, "netstat": netstat, "controller": controller, "audio_id": audio_id, "key_id": key_id, "user_id": user_id } else: if not self.is_same_target(adjust_log_info, device_last_log): continue if action == "seek_start": last_end_ts = ts last_end_type = "seek_stop" last_end_pos = start_time elif action == "pause": last_end_ts = ts last_end_type = "pause" last_end_pos = play_time elif action == "stop": last_end_ts = ts last_end_type = "stop" last_end_pos = last_end_pos if play_time == 0 else play_time elif action == "complete": last_end_ts = ts last_end_type = "complete" last_end_pos = total_time if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if last_phid not in devices_audio_info[device_id]: devices_audio_info[device_id][last_phid] = {} devices_audio_info[device_id][last_phid][ last_start_ts] = { "start_type": last_start_type, "end_type": last_end_type, "end_time": last_end_ts, "live_status": live, "pid": last_pid, "tid": last_tid, "album_id": last_album_id, "start_pos": last_start_pos if last_start_pos is not None else last_end_pos, "end_pos": last_end_pos, "netstat": netstat if last_netstat is None else last_netstat, "controller": controller if last_controller is None else last_controller, "audio_id": last_audio_id, "key_id": last_key_id, "user_id": last_user_id } del self.device_audio_record[device_id] if action == "seek_start": start_type = "seek_start" if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if phid not in devices_audio_info[device_id]: devices_audio_info[device_id][phid] = {} devices_audio_info[device_id][phid][ts] = { "start_type": start_type, "end_type": None, "end_time": None, "live_status": live, "pid": pid, "tid": tid, "album_id": album_id, "start_pos": target_time, "end_pos": None, "netstat": netstat, "controller": controller, "audio_id": audio_id, "key_id": key_id, "user_id": user_id } self.device_audio_record[device_id] = { "action": action, "start_type": start_type, "start_ts": ts, "update_ts": ts, "phid": phid, "pid": pid, "tid": tid, "album_id": album_id, "live": live, "start_pos": target_time, "end_pos": None, "netstat": netstat, "controller": controller, "audio_id": audio_id, "key_id": key_id, "user_id": user_id } else: if action in ["buffer", "buffer_full"]: pass else: diff_ts = int( (ts - last_update_ts).total_seconds()) if diff_ts >= 40: last_end_ts = last_update_ts + timedelta( seconds=30) last_end_type = "stop" if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if last_phid not in devices_audio_info[ device_id]: devices_audio_info[device_id][ last_phid] = {} devices_audio_info[device_id][last_phid][ last_start_ts] = { "start_type": last_start_type, "end_type": last_end_type, "end_time": last_end_ts, "live_status": live, "pid": last_pid, "tid": last_tid, "album_id": last_album_id, "start_pos": last_start_pos if last_start_pos is not None else last_end_pos, "end_pos": last_end_pos, "netstat": netstat if last_netstat is None else last_netstat, "controller": controller if last_controller is None else last_controller, "audio_id": last_audio_id, "key_id": last_key_id, "user_id": last_user_id } del self.device_audio_record[device_id] start_type = "start" if device_id not in devices_audio_info: devices_audio_info[device_id] = {} if phid not in devices_audio_info[device_id]: devices_audio_info[device_id][phid] = {} devices_audio_info[device_id][phid][ts] = { "start_type": start_type, "end_type": None, "end_time": None, "live_status": live, "pid": pid, "tid": tid, "album_id": album_id, "start_pos": play_time, "end_pos": None, "netstat": netstat, "controller": controller, "audio_id": audio_id, "key_id": key_id, "user_id": user_id } self.device_audio_record[device_id] = { "action": action, "start_type": start_type, "start_ts": ts, "update_ts": ts, "phid": phid, "pid": pid, "tid": tid, "album_id": album_id, "live": live, "start_pos": play_time, "end_pos": None, "netstat": netstat, "controller": controller, "audio_id": audio_id, "key_id": key_id, "user_id": user_id } else: self.device_audio_record[device_id] = { "action": action, "start_type": last_start_type, "start_ts": last_start_ts, "update_ts": ts, "phid": last_phid, "pid": last_pid, "tid": last_tid, "album_id": last_album_id, "live": live, "start_pos": last_start_pos if last_start_pos is not None else play_time, "end_pos": play_time, "netstat": netstat if last_netstat is None else last_netstat, "controller": controller if last_controller is None else last_controller, "audio_id": last_audio_id, "key_id": last_key_id, "user_id": last_user_id } if device_id in devices_audio_info: if last_phid in devices_audio_info[ device_id]: if last_start_ts in devices_audio_info[ device_id][last_phid]: if devices_audio_info[device_id][ last_phid][last_start_ts][ "live_status"] is None: devices_audio_info[device_id][ last_phid][last_start_ts][ "live_status"] = live if devices_audio_info[device_id][ last_phid][last_start_ts][ "start_pos"] is None: devices_audio_info[device_id][ last_phid][last_start_ts][ "start_pos"] = play_time if devices_audio_info[device_id][ last_phid][last_start_ts][ "controller"] is None: devices_audio_info[device_id][ last_phid][last_start_ts][ "controller"] = controller return devices_audio_info def is_same_target(self, a, b): for k in ["phid", "pid", "user_id", "tid", "live", "album_id"]: if k == "live" and b.get(k) is None: continue if a.get(k) != b.get(k): return False return True def adjust_logs(self, lines): lines = sorted(lines, key=lambda l: l.get("log_time")) adjust_lines = [] while len(lines) > 1: a = lines[0] b = lines[1] result = self.action_sort(a, b) adjust_lines.append(result[0]) lines = lines[1:] lines[0] = result[1] adjust_lines.append(lines[0]) return adjust_lines def action_sort(self, a, b): if a.get("log_time") != b.get("log_time"): return (a, b) a_action = a.get("params_info").get("t2").lower() if a.get( "params_info").get("t2") is not None else None b_action = b.get("params_info").get("t2").lower() if b.get( "params_info").get("t2") is not None else None if (a_action, b_action) in self.action_adjust_list: return (a, b) if (b_action, a_action) in self.action_adjust_list: return (b, a) return (a, b) def get_devices_logs(self, logs_info): devices_logs = {} for log_info in logs_info: device = log_info.get("device") if device not in devices_logs: devices_logs[device] = [] devices_logs[device].append(log_info) return devices_logs
class CalculateAudioDataV1(object): def __init__(self, status_value, delay_value, data_record, stop_flag): self.status_value = status_value self.status_value.set(1) self.delay_value = delay_value self.data_record = data_record self.stop_flag = stop_flag config_obj = ConfigReader() config_key = "audio_queue" audio_queue_config = config_obj.get_section_config(config_key) self.queue_ip = audio_queue_config.get("host") self.queue_port = int(audio_queue_config.get("port")) self.queue_key = audio_queue_config.get("key").encode() config_key = "cassandra" cassandra_config = config_obj.get_section_config(config_key) cassandra_host = [ host.strip() for host in cassandra_config.get("hosts").split(',') ] cassandra_user = cassandra_config.get("username") cassandra_password = cassandra_config.get("password") self.cassandra_connector = CassandraConnector() self.cassandra_connector.db = cassandra_host self.cassandra_connector.username = cassandra_user self.cassandra_connector.password = cassandra_password self.audio_queue = self.get_audio_queue() if self.audio_queue is False: logging.error("get audio queue error") sys.exit(1) self.period_logs_info = {} self.device_audio_record = Manager().dict() @function_retry(retry_times=RETRY_TIMES, period_seconds=PERIOD_SECONDS) def get_audio_queue(self): BaseManager.register("audio_queue_v1") manager = BaseManager(address=(self.queue_ip, self.queue_port), authkey=self.queue_key) manager.connect() return manager.audio_queue_v1() def init_cassandra(self): if not self.cassandra_connector.prepare_stmt( "update_device_audio_data", "data_dictionary", "update device_audio_daily set start_type=?,end_type=?,end_time=?,live_status=?,pid=?,tid=?,netstat=?,start_pos=?,end_pos=? where date=? and device_id=? and start_time=? and phid=?" ): logging.error("init cassandra db error") sys.exit(2) def run(self): self.status_value.set(2) self.init_cassandra() self.status_value.set(3) thread_get_logs = threading.Thread(target=self.get_logs_from_queue) thread_calculate_logs = threading.Thread(target=self.calculate_logs) thread_get_logs.start() thread_calculate_logs.start() thread_get_logs.join() thread_calculate_logs.join() self.status_value.set(4) for period_key in self.period_logs_info: self.data_record[period_key] = self.period_logs_info[period_key] self.status_value.set(-1) def calculate_logs(self): while not self.stop_flag.get() == 1: now_ts = datetime.now().replace(microsecond=0) period_key_list = sorted(self.period_logs_info.keys()) calculate_logs_info = {} for period_key in period_key_list: if self.period_logs_info[period_key].get("end_ts") is None: continue # """离线计算""" # if period_key.hour == 23 and period_key.minute == 59: # self.stop_flag.set(1) # calculate_logs_info[period_key] = self.period_logs_info[period_key].get("log_list") # del self.period_logs_info[period_key] # break # else: # calculate_logs_info[period_key] = self.period_logs_info[period_key].get("log_list") # del self.period_logs_info[period_key] """实时计算""" if now_ts > self.period_logs_info[period_key].get("end_ts"): if period_key.hour == 23 and period_key.minute == 59: self.stop_flag.set(1) calculate_logs_info[ period_key] = self.period_logs_info[ period_key].get("log_list") del self.period_logs_info[period_key] break else: calculate_logs_info[ period_key] = self.period_logs_info[ period_key].get("log_list") del self.period_logs_info[period_key] calculate_minute_list = sorted(calculate_logs_info.keys()) for calculate_minute_item in calculate_minute_list: logging.info("start calculate minute v1: {0}".format( calculate_minute_item.strftime("%Y-%m-%d %H:%M:00"))) start_ts = time.time() calculate_one_minute_logs_obj = CalculateOneMinuteLogsV1( self.device_audio_record) calculate_one_minute_logs_process = Process( target=calculate_one_minute_logs_obj.run, args=(calculate_logs_info[calculate_minute_item], )) calculate_one_minute_logs_process.start() calculate_one_minute_logs_process.join() try: self.check_device_audio_record(calculate_minute_item + timedelta(seconds=59)) except: error_data = traceback.format_exc() logging.error( "check device audio record error: {0}".format( error_data)) if calculate_minute_item.hour == 23 and calculate_minute_item.minute == 59: try: self.end_device_audio_record(calculate_minute_item + timedelta(seconds=59)) except: error_data = traceback.format_exc() logging.error( "end device audio record error: {0}".format( error_data)) end_ts = time.time() logging.info( "calculate minute v1: {0} over, it takes: {1} seconds". format(calculate_minute_item, int(end_ts - start_ts))) def end_device_audio_record(self, end_ts): device_id_list = list(self.device_audio_record.keys()) jobs = {} job_index = 0 for device_id in device_id_list: device_info = self.device_audio_record[device_id] last_action = device_info.get("action") last_update_ts = device_info.get("update_ts") last_start_type = device_info.get("start_type") last_start_ts = device_info.get("start_ts") last_phid = device_info.get("phid") last_live = device_info.get("live") last_pid = device_info.get("pid") last_tid = device_info.get("tid") last_netstat = device_info.get("netstat") last_start_pos = device_info.get("start_pos") last_end_pos = device_info.get("end_pos") diff_ts = int((end_ts - last_update_ts).total_seconds()) if last_action == "pause": last_end_ts = last_update_ts last_end_type = "pause" else: if diff_ts >= 40: last_end_ts = last_update_ts + timedelta(seconds=30) else: last_end_ts = end_ts last_end_type = "stop" if last_end_ts is not None and last_end_ts < last_start_ts: last_end_ts = last_start_ts + timedelta(seconds=30) jobs[job_index] = [ last_start_type, last_end_type, convert_into_utc_time(last_end_ts) if last_end_ts is not None else None, last_live, last_pid, last_tid, last_netstat, last_start_pos, last_end_pos, convert_into_utc_time( last_start_ts.replace(second=0, minute=0, hour=0)), device_id, convert_into_utc_time(last_start_ts), last_phid ] job_index += 1 del self.device_audio_record[device_id] if len(jobs) > 0: self.cassandra_connector.do_futures("update_device_audio_data", jobs) def check_device_audio_record(self, check_ts): device_id_list = list(self.device_audio_record.keys()) jobs = {} job_index = 0 for device_id in device_id_list: device_info = self.device_audio_record[device_id] last_action = device_info.get("action") last_update_ts = device_info.get("update_ts") last_start_type = device_info.get("start_type") last_start_ts = device_info.get("start_ts") last_phid = device_info.get("phid") last_live = device_info.get("live") last_pid = device_info.get("pid") last_tid = device_info.get("tid") last_netstat = device_info.get("netstat") last_start_pos = device_info.get("start_pos") last_end_pos = device_info.get("end_pos") diff_ts = int((check_ts - last_update_ts).total_seconds()) if diff_ts >= 40: if last_action == "pause": last_end_ts = last_update_ts last_end_type = "pause" else: last_end_ts = last_update_ts + timedelta(seconds=30) last_end_type = "stop" if last_end_ts is not None and last_end_ts < last_start_ts: last_end_ts = last_start_ts + timedelta(seconds=30) jobs[job_index] = [ last_start_type, last_end_type, convert_into_utc_time(last_end_ts) if last_end_ts is not None else None, last_live, last_pid, last_tid, last_netstat, last_start_pos, last_end_pos, convert_into_utc_time( last_start_ts.replace(second=0, minute=0, hour=0)), device_id, convert_into_utc_time(last_start_ts), last_phid ] job_index += 1 del self.device_audio_record[device_id] if len(jobs) > 0: self.cassandra_connector.do_futures("update_device_audio_data", jobs) def get_logs_from_queue(self): for data_key in self.data_record.keys(): self.period_logs_info[data_key] = self.data_record.get(data_key) while not self.stop_flag.get() == 1: try: if not self.audio_queue.empty(): try: now_ts = datetime.now().replace(microsecond=0) message = self.audio_queue.get() log_info = json.loads(message.decode("utf-8"), object_hook=date_decoder) log_time = log_info.get("log_time") log_minute = log_time.replace(second=0) log_end_ts = log_minute + timedelta( minutes=1) + timedelta(seconds=self.delay_value) # """离线计算""" # if log_minute not in self.period_logs_info: # self.period_logs_info[log_minute] = {"log_list": [], "end_ts": log_end_ts} # # self.period_logs_info[log_minute]["log_list"].append(log_info) """实时计算""" if now_ts <= log_end_ts: if log_minute not in self.period_logs_info: self.period_logs_info[log_minute] = { "log_list": [], "end_ts": log_end_ts } self.period_logs_info[log_minute][ "log_list"].append(log_info) except: logging.error(message) else: logging.info("audio queue is empty...") time.sleep(1) except: logging.error("audio queue is broken!!!") break