class CalculateOneMinuteLogs(object):
def __init__(self, device_audio_record):
self.device_audio_record = device_audio_record
config_obj = ConfigReader()
config_key = "cassandra"
cassandra_config = config_obj.get_section_config(config_key)
cassandra_host = [
host.strip() for host in cassandra_config.get("hosts").split(',')
]
cassandra_user = cassandra_config.get("username")
cassandra_password = cassandra_config.get("password")
self.cassandra_connector = CassandraConnector()
self.cassandra_connector.db = cassandra_host
self.cassandra_connector.username = cassandra_user
self.cassandra_connector.password = cassandra_password
def init_cassandra(self):
if not self.cassandra_connector.prepare_stmt(
"update_device_audio_data", "data_dictionary",
"update device_audio_daily set start_type=?,end_type=?,end_time=?,live_status=?,pid=?,tid=?,netstat=?,start_pos=?,end_pos=?,album_id=?,controller=?,audio_id=?,key_id=?,user_id=? where date=? and device_id=? and start_time=? and phid=?"
):
logging.error("init cassandra db error")
sys.exit(1)
def run(self, calculate_logs):
self.init_cassandra()
self.action_adjust_list = [("stop", "start"), ("complete", "start"),
("start", "play"), ("play", "complete"),
("play", "stop"), ("play", "pause"),
("resume", "play"), ("pause", "resume"),
("seek_start", "start")]
result = self.calculate_audio_logs(calculate_logs)
jobs = {}
job_index = 0
if result is not None:
for device_id in result:
for phid in result[device_id]:
for start_time in result[device_id][phid]:
end_type = result[device_id][phid][start_time][
"end_type"]
end_time = result[device_id][phid][start_time][
"end_time"]
start_type = result[device_id][phid][start_time][
"start_type"]
live_status = result[device_id][phid][start_time][
"live_status"]
pid = result[device_id][phid][start_time]["pid"]
tid = result[device_id][phid][start_time]["tid"]
album_id = result[device_id][phid][start_time][
"album_id"]
netstat = result[device_id][phid][start_time][
"netstat"]
controller = result[device_id][phid][start_time][
"controller"]
start_pos = result[device_id][phid][start_time][
"start_pos"]
end_pos = result[device_id][phid][start_time][
"end_pos"]
audio_id = result[device_id][phid][start_time][
"audio_id"]
key_id = result[device_id][phid][start_time]["key_id"]
user_id = result[device_id][phid][start_time][
"user_id"]
if end_time is not None and end_time < start_time:
end_time = start_time + timedelta(seconds=30)
jobs[job_index] = [
start_type, end_type,
convert_into_utc_time(end_time)
if end_time is not None else None, live_status,
pid, tid, netstat, start_pos, end_pos, album_id,
controller, audio_id, key_id, user_id,
convert_into_utc_time(
start_time.replace(second=0, minute=0,
hour=0)), device_id,
convert_into_utc_time(start_time), phid
]
job_index += 1
if len(jobs) > 0:
try:
self.cassandra_connector.do_futures("update_device_audio_data",
jobs)
except:
error_data = traceback.format_exc()
logging.error(
"update device audio data error: {0}".format(error_data))
def calculate_audio_logs(self, audio_logs):
devices_logs = self.get_devices_logs(audio_logs)
if len(devices_logs) == 0:
return None
devices_audio_info = self.get_devices_audio_info(devices_logs)
return devices_audio_info
def get_devices_audio_info(self, devices_logs):
devices_audio_info = {}
for device_id in devices_logs:
device_logs = self.adjust_logs(devices_logs[device_id])
for log_info in device_logs:
action = log_info.get("params_info").get("t2")
ts = log_info.get("log_time")
phid = log_info.get("params_info").get("phid")
if action is None or phid is None:
continue
try:
phid = int(phid)
except:
logging.error("the wrong phid: {0}".format(phid))
continue
action = action.lower()
try:
pid = int(log_info.get("params_info").get("pid"))
except:
# logging.warning("the wrong pid: {0}".format(log_info.get("params_info").get("pid")))
pid = 0
try:
tid = int(log_info.get("params_info").get("tid"))
except:
# logging.warning("the wrong tid: {0}".format(log_info.get("params_info").get("tid")))
tid = 0
try:
live = int(log_info.get("params_info").get("islive"))
except:
# logging.warning("the wrong live status: {0}".format(log_info.get("params_info").get("islive")))
live = 0
try:
album_id = int(log_info.get("params_info").get("albumId"))
except:
# logging.warning("the wrong album id: {0}".format(log_info.get("params_info").get("albumId")))
album_id = 0
try:
audio_id = int(log_info.get("params_info").get("audioId"))
except:
audio_id = None
try:
key_id = int(log_info.get("params_info").get("keyId"))
except:
key_id = None
try:
user_id = int(log_info.get("params_info").get("userId"))
except:
user_id = 0
play_time = int(
round(float(log_info.get("params_info").get("playtime")),
0)) if log_info.get("params_info").get(
"playtime") is not None else None
start_time = int(
round(float(log_info.get("params_info").get("startTime")),
0)) if log_info.get("params_info").get(
"startTime") is not None else None
target_time = int(
round(float(log_info.get("params_info").get("targetTime")),
0)) if log_info.get("params_info").get(
"targetTime") is not None else None
total_time = int(
round(float(log_info.get("params_info").get("totalTime")),
0)) if log_info.get("params_info").get(
"totalTime") is not None else None
netstat = log_info.get("params_info").get("netstat")
controller = log_info.get("params_info").get("controller")
if live < 0:
live = 0
adjust_log_info = {
"phid": phid,
"pid": pid,
"user_id": user_id,
"tid": tid,
"live": live,
"album_id": album_id
}
if device_id not in self.device_audio_record:
if action not in [
"stop", "pause", "complete", "buffer",
"buffer_full"
]:
if action == "resume":
start_type = "resume"
start_pos = play_time
elif action == "seek_start":
start_type = "seek_start"
start_pos = target_time
elif action == "play":
start_type = "play"
start_pos = play_time
else:
start_type = "start"
start_pos = None
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if phid not in devices_audio_info[device_id]:
devices_audio_info[device_id][phid] = {}
devices_audio_info[device_id][phid][ts] = {
"start_type": start_type,
"end_type": None,
"end_time": None,
"live_status":
None if action in ["start"] else live,
"pid": pid,
"tid": tid,
"album_id": album_id,
"start_pos": start_pos,
"end_pos": None,
"netstat": netstat,
"controller": controller,
"audio_id": audio_id,
"key_id": key_id,
"user_id": user_id
}
self.device_audio_record[device_id] = {
"action": action,
"start_type": start_type,
"start_ts": ts,
"update_ts": ts,
"phid": phid,
"pid": pid,
"tid": tid,
"album_id": album_id,
"audio_id": audio_id,
"key_id": key_id,
"user_id": user_id,
"live": None if action in ["start"] else live,
"start_pos": start_pos,
"end_pos": None,
"netstat": netstat,
"controller": controller
}
else:
device_last_log = self.device_audio_record[device_id]
last_action = device_last_log.get("action")
last_update_ts = device_last_log.get("update_ts")
last_start_type = device_last_log.get("start_type")
last_start_ts = device_last_log.get("start_ts")
last_phid = device_last_log.get("phid")
last_live = device_last_log.get("live")
last_pid = device_last_log.get("pid")
last_tid = device_last_log.get("tid")
last_album_id = device_last_log.get("album_id")
last_start_pos = device_last_log.get("start_pos")
last_end_pos = device_last_log.get("end_pos")
last_netstat = device_last_log.get("netstat")
last_controller = device_last_log.get("controller")
last_audio_id = device_last_log.get("audio_id")
last_key_id = device_last_log.get("key_id")
last_user_id = device_last_log.get("user_id")
if action in [
"start", "seek_start", "pause", "stop", "complete"
]:
if action == "start":
diff_ts = int(ts.timestamp() -
last_update_ts.timestamp())
if diff_ts >= 40:
last_end_ts = last_update_ts + timedelta(
seconds=30)
else:
last_end_ts = ts
last_end_type = "stop"
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if last_phid not in devices_audio_info[device_id]:
devices_audio_info[device_id][last_phid] = {}
devices_audio_info[device_id][last_phid][
last_start_ts] = {
"start_type": last_start_type,
"end_type": last_end_type,
"end_time": last_end_ts,
"live_status": last_live,
"pid": last_pid,
"tid": last_tid,
"album_id": last_album_id,
"start_pos": last_start_pos,
"end_pos": last_end_pos,
"netstat": last_netstat,
"controller": last_controller,
"audio_id": last_audio_id,
"key_id": last_key_id,
"user_id": last_user_id
}
del self.device_audio_record[device_id]
start_type = "start"
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if phid not in devices_audio_info[device_id]:
devices_audio_info[device_id][phid] = {}
devices_audio_info[device_id][phid][ts] = {
"start_type": start_type,
"end_type": None,
"end_time": None,
"live_status": None,
"pid": pid,
"tid": tid,
"album_id": album_id,
"start_pos": None,
"end_pos": None,
"netstat": netstat,
"controller": controller,
"audio_id": audio_id,
"key_id": key_id,
"user_id": user_id
}
self.device_audio_record[device_id] = {
"action": action,
"start_type": start_type,
"start_ts": ts,
"update_ts": ts,
"phid": phid,
"pid": pid,
"tid": tid,
"album_id": album_id,
"live": None,
"start_pos": None,
"end_pos": None,
"netstat": netstat,
"controller": controller,
"audio_id": audio_id,
"key_id": key_id,
"user_id": user_id
}
else:
if not self.is_same_target(adjust_log_info,
device_last_log):
continue
if action == "seek_start":
last_end_ts = ts
last_end_type = "seek_stop"
last_end_pos = start_time
elif action == "pause":
last_end_ts = ts
last_end_type = "pause"
last_end_pos = play_time
elif action == "stop":
last_end_ts = ts
last_end_type = "stop"
last_end_pos = last_end_pos if play_time == 0 else play_time
elif action == "complete":
last_end_ts = ts
last_end_type = "complete"
last_end_pos = total_time
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if last_phid not in devices_audio_info[device_id]:
devices_audio_info[device_id][last_phid] = {}
devices_audio_info[device_id][last_phid][
last_start_ts] = {
"start_type":
last_start_type,
"end_type":
last_end_type,
"end_time":
last_end_ts,
"live_status":
live,
"pid":
last_pid,
"tid":
last_tid,
"album_id":
last_album_id,
"start_pos":
last_start_pos if last_start_pos
is not None else last_end_pos,
"end_pos":
last_end_pos,
"netstat":
netstat
if last_netstat is None else last_netstat,
"controller":
controller if last_controller is None else
last_controller,
"audio_id":
last_audio_id,
"key_id":
last_key_id,
"user_id":
last_user_id
}
del self.device_audio_record[device_id]
if action == "seek_start":
start_type = "seek_start"
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if phid not in devices_audio_info[device_id]:
devices_audio_info[device_id][phid] = {}
devices_audio_info[device_id][phid][ts] = {
"start_type": start_type,
"end_type": None,
"end_time": None,
"live_status": live,
"pid": pid,
"tid": tid,
"album_id": album_id,
"start_pos": target_time,
"end_pos": None,
"netstat": netstat,
"controller": controller,
"audio_id": audio_id,
"key_id": key_id,
"user_id": user_id
}
self.device_audio_record[device_id] = {
"action": action,
"start_type": start_type,
"start_ts": ts,
"update_ts": ts,
"phid": phid,
"pid": pid,
"tid": tid,
"album_id": album_id,
"live": live,
"start_pos": target_time,
"end_pos": None,
"netstat": netstat,
"controller": controller,
"audio_id": audio_id,
"key_id": key_id,
"user_id": user_id
}
else:
if action in ["buffer", "buffer_full"]:
pass
else:
diff_ts = int(
(ts - last_update_ts).total_seconds())
if diff_ts >= 40:
last_end_ts = last_update_ts + timedelta(
seconds=30)
last_end_type = "stop"
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if last_phid not in devices_audio_info[
device_id]:
devices_audio_info[device_id][
last_phid] = {}
devices_audio_info[device_id][last_phid][
last_start_ts] = {
"start_type":
last_start_type,
"end_type":
last_end_type,
"end_time":
last_end_ts,
"live_status":
live,
"pid":
last_pid,
"tid":
last_tid,
"album_id":
last_album_id,
"start_pos":
last_start_pos if last_start_pos
is not None else last_end_pos,
"end_pos":
last_end_pos,
"netstat":
netstat if last_netstat is None else
last_netstat,
"controller":
controller if last_controller is None
else last_controller,
"audio_id":
last_audio_id,
"key_id":
last_key_id,
"user_id":
last_user_id
}
del self.device_audio_record[device_id]
start_type = "start"
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if phid not in devices_audio_info[device_id]:
devices_audio_info[device_id][phid] = {}
devices_audio_info[device_id][phid][ts] = {
"start_type": start_type,
"end_type": None,
"end_time": None,
"live_status": live,
"pid": pid,
"tid": tid,
"album_id": album_id,
"start_pos": play_time,
"end_pos": None,
"netstat": netstat,
"controller": controller,
"audio_id": audio_id,
"key_id": key_id,
"user_id": user_id
}
self.device_audio_record[device_id] = {
"action": action,
"start_type": start_type,
"start_ts": ts,
"update_ts": ts,
"phid": phid,
"pid": pid,
"tid": tid,
"album_id": album_id,
"live": live,
"start_pos": play_time,
"end_pos": None,
"netstat": netstat,
"controller": controller,
"audio_id": audio_id,
"key_id": key_id,
"user_id": user_id
}
else:
self.device_audio_record[device_id] = {
"action":
action,
"start_type":
last_start_type,
"start_ts":
last_start_ts,
"update_ts":
ts,
"phid":
last_phid,
"pid":
last_pid,
"tid":
last_tid,
"album_id":
last_album_id,
"live":
live,
"start_pos":
last_start_pos if last_start_pos
is not None else play_time,
"end_pos":
play_time,
"netstat":
netstat
if last_netstat is None else last_netstat,
"controller":
controller if last_controller is None else
last_controller,
"audio_id":
last_audio_id,
"key_id":
last_key_id,
"user_id":
last_user_id
}
if device_id in devices_audio_info:
if last_phid in devices_audio_info[
device_id]:
if last_start_ts in devices_audio_info[
device_id][last_phid]:
if devices_audio_info[device_id][
last_phid][last_start_ts][
"live_status"] is None:
devices_audio_info[device_id][
last_phid][last_start_ts][
"live_status"] = live
if devices_audio_info[device_id][
last_phid][last_start_ts][
"start_pos"] is None:
devices_audio_info[device_id][
last_phid][last_start_ts][
"start_pos"] = play_time
if devices_audio_info[device_id][
last_phid][last_start_ts][
"controller"] is None:
devices_audio_info[device_id][
last_phid][last_start_ts][
"controller"] = controller
return devices_audio_info
def is_same_target(self, a, b):
for k in ["phid", "pid", "user_id", "tid", "live", "album_id"]:
if k == "live" and b.get(k) is None:
continue
if a.get(k) != b.get(k):
return False
return True
def adjust_logs(self, lines):
lines = sorted(lines, key=lambda l: l.get("log_time"))
adjust_lines = []
while len(lines) > 1:
a = lines[0]
b = lines[1]
result = self.action_sort(a, b)
adjust_lines.append(result[0])
lines = lines[1:]
lines[0] = result[1]
adjust_lines.append(lines[0])
return adjust_lines
def action_sort(self, a, b):
if a.get("log_time") != b.get("log_time"):
return (a, b)
a_action = a.get("params_info").get("t2").lower() if a.get(
"params_info").get("t2") is not None else None
b_action = b.get("params_info").get("t2").lower() if b.get(
"params_info").get("t2") is not None else None
if (a_action, b_action) in self.action_adjust_list:
return (a, b)
if (b_action, a_action) in self.action_adjust_list:
return (b, a)
return (a, b)
def get_devices_logs(self, logs_info):
devices_logs = {}
for log_info in logs_info:
device = log_info.get("device")
if device not in devices_logs:
devices_logs[device] = []
devices_logs[device].append(log_info)
return devices_logs
class CalculateOneMinuteLogs(object):
def __init__(self, device_audio_record):
self.device_audio_record = device_audio_record
config_obj = ConfigReader()
config_key = "cassandra"
cassandra_config = config_obj.get_section_config(config_key)
cassandra_host = [
host.strip() for host in cassandra_config.get("hosts").split(',')
]
cassandra_user = cassandra_config.get("username")
cassandra_password = cassandra_config.get("password")
self.cassandra_connector = CassandraConnector()
self.cassandra_connector.db = cassandra_host
self.cassandra_connector.username = cassandra_user
self.cassandra_connector.password = cassandra_password
def init_cassandra(self):
if not self.cassandra_connector.prepare_stmt(
"update_device_audio_data", "data_dictionary",
"update device_audio_daily set start_type=?,end_type=?,end_time=?,live_status=?,pid=?,tid=?,netstat=? where date=? and device_id=? and start_time=? and phid=?"
):
logging.error("init cassandra db error")
sys.exit(1)
def run(self, calculate_logs):
self.init_cassandra()
result = self.calculate_audio_logs(calculate_logs)
jobs = {}
job_index = 0
if result is not None:
for device_id in result:
for phid in result[device_id]:
for start_time in result[device_id][phid]:
end_type = result[device_id][phid][start_time][
"end_type"]
end_time = result[device_id][phid][start_time][
"end_time"]
start_type = result[device_id][phid][start_time][
"start_type"]
live_status = result[device_id][phid][start_time][
"live_status"]
pid = result[device_id][phid][start_time]["pid"]
tid = result[device_id][phid][start_time]["tid"]
netstat = None
if end_time is not None and end_time < start_time:
end_time = start_time + timedelta(seconds=30)
jobs[job_index] = [
start_type, end_type,
convert_into_utc_time(end_time)
if end_time is not None else None, live_status,
pid, tid, netstat,
convert_into_utc_time(
start_time.replace(second=0, minute=0,
hour=0)), device_id,
convert_into_utc_time(start_time), phid
]
job_index += 1
if len(jobs) > 0:
try:
self.cassandra_connector.do_futures("update_device_audio_data",
jobs)
except:
error_data = traceback.format_exc()
logging.error(
"update device audio data error: {0}".format(error_data))
def calculate_audio_logs(self, audio_logs):
devices_logs = self.get_devices_logs(audio_logs)
if len(devices_logs) == 0:
return None
devices_audio_info = self.get_devices_audio_info(devices_logs)
return devices_audio_info
def get_devices_audio_info(self, devices_logs):
devices_audio_info = {}
for device_id in devices_logs:
device_logs = self.adjust_logs(devices_logs[device_id])
for log_info in device_logs:
action = log_info.get("action")
ts = log_info.get("ts")
phid = log_info.get("phid")
if action is None or phid is None:
continue
try:
phid = int(phid)
except:
logging.error("the wrong phid: {0}".format(phid))
continue
try:
pid = int(log_info.get("pid"))
except:
# logging.warning("the wrong pid: {0}".format(log_info.get("pid")))
pid = 0
try:
tid = int(log_info.get("tid"))
except:
# logging.warning("the wrong tid: {0}".format(log_info.get("tid")))
tid = 0
try:
live = int(log_info.get("live"))
except:
# logging.warning("the wroing live status: {0}".format(log_info.get("live")))
live = 0
if live < 0:
live = 0
adjust_log_info = {
"phid": phid,
"pid": pid,
"tid": tid,
"live": live
}
if device_id not in self.device_audio_record:
if action not in ["stop", "pause", "complete"]:
start_type = "start"
self.device_audio_record[device_id] = {
"action": action,
"start_type": start_type,
"start_ts": ts,
"update_ts": ts,
"phid": phid,
"pid": pid,
"tid": tid,
"live":
None if action in ["start", "buffer"] else live
}
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if phid not in devices_audio_info[device_id]:
devices_audio_info[device_id][phid] = {}
devices_audio_info[device_id][phid][ts] = {
"start_type":
start_type,
"end_type":
None,
"end_time":
None,
"live_status":
None if action in ["start", "buffer"] else live,
"pid":
pid,
"tid":
tid
}
else:
device_last_log = self.device_audio_record[device_id]
last_action = device_last_log.get("action")
last_update_ts = device_last_log.get("update_ts")
last_start_type = device_last_log.get("start_type")
last_start_ts = device_last_log.get("start_ts")
last_phid = device_last_log.get("phid")
last_live = device_last_log.get("live")
last_pid = device_last_log.get("pid")
last_tid = device_last_log.get("tid")
if action in ["start", "stop", "complete"]:
if last_action == "pause":
last_end_ts = last_update_ts
last_end_type = "pause"
else:
if action == "start":
diff_ts = int(ts.timestamp() -
last_update_ts.timestamp())
if diff_ts >= 40:
last_end_ts = last_update_ts + timedelta(
seconds=30)
else:
last_end_ts = ts
last_end_type = "stop"
else:
last_end_ts = ts
if action == "complete":
last_end_type = "complete"
else:
last_end_type = "stop"
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if last_phid not in devices_audio_info[device_id]:
devices_audio_info[device_id][last_phid] = {}
devices_audio_info[device_id][last_phid][
last_start_ts] = {
"start_type": last_start_type,
"end_type": last_end_type,
"end_time": last_end_ts,
"live_status": last_live,
"pid": last_pid,
"tid": last_tid
}
del self.device_audio_record[device_id]
if action == "start":
self.device_audio_record[device_id] = {
"action":
action,
"start_type":
"start",
"start_ts":
ts,
"update_ts":
ts,
"phid":
phid,
"pid":
pid,
"tid":
tid,
"live":
None if action in ["start", "buffer"] else live
}
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if phid not in devices_audio_info[device_id]:
devices_audio_info[device_id][phid] = {}
devices_audio_info[device_id][phid][ts] = {
"start_type":
"start",
"end_type":
None,
"end_time":
None,
"live_status":
None
if action in ["start", "buffer"] else live,
"pid":
pid,
"tid":
tid
}
elif action == "pause":
last_end_ts = ts
last_end_type = "pause"
self.device_audio_record[device_id] = {
"action": action,
"start_type": last_start_type,
"start_ts": last_start_ts,
"update_ts": ts,
"phid": last_phid,
"pid": last_pid,
"tid": last_tid,
"live": live
}
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if last_phid not in devices_audio_info[device_id]:
devices_audio_info[device_id][last_phid] = {}
devices_audio_info[device_id][last_phid][
last_start_ts] = {
"start_type": last_start_type,
"end_type": last_end_type,
"end_time": last_end_ts,
"live_status": live,
"pid": last_pid,
"tid": last_tid
}
else:
if not self.is_same_target(adjust_log_info,
device_last_log):
if action == "buffer":
pass
else:
if last_action == "pause":
last_end_ts = last_update_ts
last_end_type = "pause"
else:
diff_ts = int(
(ts - last_update_ts).total_seconds())
if diff_ts >= 40:
last_end_ts = last_update_ts + timedelta(
seconds=30)
else:
last_end_ts = ts
last_end_type = "stop"
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if last_phid not in devices_audio_info[
device_id]:
devices_audio_info[device_id][
last_phid] = {}
devices_audio_info[device_id][last_phid][
last_start_ts] = {
"start_type": last_start_type,
"end_type": last_end_type,
"end_time": last_end_ts,
"live_status": last_live,
"pid": last_pid,
"tid": last_tid
}
self.device_audio_record[device_id] = {
"action": action,
"start_type": "start",
"start_ts": ts,
"update_ts": ts,
"phid": phid,
"pid": pid,
"tid": tid,
"live": live
}
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if phid not in devices_audio_info[device_id]:
devices_audio_info[device_id][phid] = {}
devices_audio_info[device_id][phid][ts] = {
"start_type": "start",
"end_type": None,
"end_time": None,
"live_status": live,
"pid": pid,
"tid": tid
}
else:
if action == "buffer":
self.device_audio_record[device_id] = {
"action": action,
"start_type": last_start_type,
"start_ts": last_start_ts,
"update_ts": ts,
"phid": last_phid,
"pid": last_pid,
"tid": last_tid,
"live": last_live
}
else:
diff_ts = int(
(ts - last_update_ts).total_seconds())
if diff_ts >= 40:
if last_action == "pause":
last_end_ts = last_update_ts
last_end_type = "pause"
else:
last_end_ts = last_update_ts + timedelta(
seconds=30)
last_end_type = "stop"
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if last_phid not in devices_audio_info[
device_id]:
devices_audio_info[device_id][
last_phid] = {}
devices_audio_info[device_id][last_phid][
last_start_ts] = {
"start_type": last_start_type,
"end_type": last_end_type,
"end_time": last_end_ts,
"live_status": last_live,
"pid": last_pid,
"tid": last_tid
}
self.device_audio_record[device_id] = {
"action": action,
"start_type": "start",
"start_ts": ts,
"update_ts": ts,
"phid": phid,
"pid": pid,
"tid": tid,
"live": live
}
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if phid not in devices_audio_info[
device_id]:
devices_audio_info[device_id][
phid] = {}
devices_audio_info[device_id][phid][ts] = {
"start_type": "start",
"end_type": None,
"end_time": None,
"live_status": live,
"pid": pid,
"tid": tid
}
else:
if last_action == "pause":
start_type = "resume"
self.device_audio_record[device_id] = {
"action": action,
"start_type": start_type,
"start_ts": ts,
"update_ts": ts,
"phid": phid,
"pid": pid,
"tid": tid,
"live": live
}
if device_id not in devices_audio_info:
devices_audio_info[device_id] = {}
if phid not in devices_audio_info[
device_id]:
devices_audio_info[device_id][
phid] = {}
devices_audio_info[device_id][phid][
ts] = {
"start_type": start_type,
"end_type": None,
"end_time": None,
"live_status": live,
"pid": pid,
"tid": tid
}
else:
self.device_audio_record[device_id] = {
"action": action,
"start_type": last_start_type,
"start_ts": last_start_ts,
"update_ts": ts,
"phid": last_phid,
"pid": last_pid,
"tid": last_tid,
"live": live
}
if device_id in devices_audio_info:
if last_phid in devices_audio_info[
device_id]:
if last_start_ts in devices_audio_info[
device_id][last_phid]:
if devices_audio_info[
device_id][last_phid][
last_start_ts][
"live_status"] is None:
devices_audio_info[
device_id][last_phid][
last_start_ts][
"live_status"] = live
return devices_audio_info
def is_same_target(self, a, b):
for k in ["phid", "pid", "tid", "live"]:
if k == "live" and b.get(k) is None:
continue
if a.get(k) != b.get(k):
return False
return True
def adjust_logs(self, lines):
lines = [self.radio_extract(line) for line in lines]
for line in lines:
if line.get("client_ts") is None:
line["client_ts"] = line.get("ts")
min_server_ts = min([l.get("ts") for l in lines])
lines = sorted(lines, key=lambda l: self.action_sort(l.get("action")))
lines = sorted(lines, key=lambda l: l.get("client_ts"))
min_client_ts = min([l.get("client_ts") for l in lines])
for line in lines:
line["server_ts"] = line.get("ts")
line["ts"] = min_server_ts + (line.get("client_ts") -
min_client_ts)
return lines
def action_sort(self, action):
weight = {"start": 1, "stop": 0}
return 2 if weight.get(action) is None else weight.get(action)
def radio_extract(self, line):
params = line["params_info"]
ret = {"action": params.get("t2"), "ts": line.get("log_time")}
for k in ["pid", "phid", "live", "tid"]:
ret[k] = params.get(k)
if params.get("ts") is None:
ret["client_ts"] = line.get("log_time")
elif len(str(params.get("ts"))) >= 10:
try:
if len(str(params.get("ts"))) > 10:
ts = int(params.get("ts")) / 1000
else:
ts = int(params.get("ts"))
# ts = int(params.get("ts")) / (10 ** (len(str(params.get("ts"))) - 10))
ret["client_ts"] = datetime.fromtimestamp(ts)
except:
logging.error("got client ts {0}".format(params.get("ts")))
ret["client_ts"] = line.get("log_time")
else:
ret["client_ts"] = line.get("log_time")
return ret
def get_devices_last_log(self, devices_logs):
devices_last_log = {}
for device in devices_logs:
device_logs = sorted(devices_logs[device],
key=lambda l: l["log_time"],
reverse=True)
log_info = device_logs[0]
params_info = log_info.get("params_info")
action = params_info.get("t2")
phid = params_info.get("phid")
if action is None or phid is None:
continue
phid = int(phid)
devices_last_log[device] = {
"action": action,
"ts": log_info.get("log_time"),
"phid": phid
}
for key in ["pid", "tid", "live"]:
devices_last_log[device][key] = int(params_info.get(
key)) if params_info.get(key) is not None else 0
return devices_last_log
def get_devices_logs(self, logs_info):
devices_logs = {}
for log_info in logs_info:
device = log_info.get("device")
if device not in devices_logs:
devices_logs[device] = []
devices_logs[device].append(log_info)
return devices_logs
class CalculateAudioDataV1(object):
def __init__(self, status_value, delay_value, data_record, stop_flag):
self.status_value = status_value
self.status_value.set(1)
self.delay_value = delay_value
self.data_record = data_record
self.stop_flag = stop_flag
config_obj = ConfigReader()
config_key = "audio_queue"
audio_queue_config = config_obj.get_section_config(config_key)
self.queue_ip = audio_queue_config.get("host")
self.queue_port = int(audio_queue_config.get("port"))
self.queue_key = audio_queue_config.get("key").encode()
config_key = "cassandra"
cassandra_config = config_obj.get_section_config(config_key)
cassandra_host = [
host.strip() for host in cassandra_config.get("hosts").split(',')
]
cassandra_user = cassandra_config.get("username")
cassandra_password = cassandra_config.get("password")
self.cassandra_connector = CassandraConnector()
self.cassandra_connector.db = cassandra_host
self.cassandra_connector.username = cassandra_user
self.cassandra_connector.password = cassandra_password
self.audio_queue = self.get_audio_queue()
if self.audio_queue is False:
logging.error("get audio queue error")
sys.exit(1)
self.period_logs_info = {}
self.device_audio_record = Manager().dict()
@function_retry(retry_times=RETRY_TIMES, period_seconds=PERIOD_SECONDS)
def get_audio_queue(self):
BaseManager.register("audio_queue_v1")
manager = BaseManager(address=(self.queue_ip, self.queue_port),
authkey=self.queue_key)
manager.connect()
return manager.audio_queue_v1()
def init_cassandra(self):
if not self.cassandra_connector.prepare_stmt(
"update_device_audio_data", "data_dictionary",
"update device_audio_daily set start_type=?,end_type=?,end_time=?,live_status=?,pid=?,tid=?,netstat=?,start_pos=?,end_pos=? where date=? and device_id=? and start_time=? and phid=?"
):
logging.error("init cassandra db error")
sys.exit(2)
def run(self):
self.status_value.set(2)
self.init_cassandra()
self.status_value.set(3)
thread_get_logs = threading.Thread(target=self.get_logs_from_queue)
thread_calculate_logs = threading.Thread(target=self.calculate_logs)
thread_get_logs.start()
thread_calculate_logs.start()
thread_get_logs.join()
thread_calculate_logs.join()
self.status_value.set(4)
for period_key in self.period_logs_info:
self.data_record[period_key] = self.period_logs_info[period_key]
self.status_value.set(-1)
def calculate_logs(self):
while not self.stop_flag.get() == 1:
now_ts = datetime.now().replace(microsecond=0)
period_key_list = sorted(self.period_logs_info.keys())
calculate_logs_info = {}
for period_key in period_key_list:
if self.period_logs_info[period_key].get("end_ts") is None:
continue
# """离线计算"""
# if period_key.hour == 23 and period_key.minute == 59:
# self.stop_flag.set(1)
# calculate_logs_info[period_key] = self.period_logs_info[period_key].get("log_list")
# del self.period_logs_info[period_key]
# break
# else:
# calculate_logs_info[period_key] = self.period_logs_info[period_key].get("log_list")
# del self.period_logs_info[period_key]
"""实时计算"""
if now_ts > self.period_logs_info[period_key].get("end_ts"):
if period_key.hour == 23 and period_key.minute == 59:
self.stop_flag.set(1)
calculate_logs_info[
period_key] = self.period_logs_info[
period_key].get("log_list")
del self.period_logs_info[period_key]
break
else:
calculate_logs_info[
period_key] = self.period_logs_info[
period_key].get("log_list")
del self.period_logs_info[period_key]
calculate_minute_list = sorted(calculate_logs_info.keys())
for calculate_minute_item in calculate_minute_list:
logging.info("start calculate minute v1: {0}".format(
calculate_minute_item.strftime("%Y-%m-%d %H:%M:00")))
start_ts = time.time()
calculate_one_minute_logs_obj = CalculateOneMinuteLogsV1(
self.device_audio_record)
calculate_one_minute_logs_process = Process(
target=calculate_one_minute_logs_obj.run,
args=(calculate_logs_info[calculate_minute_item], ))
calculate_one_minute_logs_process.start()
calculate_one_minute_logs_process.join()
try:
self.check_device_audio_record(calculate_minute_item +
timedelta(seconds=59))
except:
error_data = traceback.format_exc()
logging.error(
"check device audio record error: {0}".format(
error_data))
if calculate_minute_item.hour == 23 and calculate_minute_item.minute == 59:
try:
self.end_device_audio_record(calculate_minute_item +
timedelta(seconds=59))
except:
error_data = traceback.format_exc()
logging.error(
"end device audio record error: {0}".format(
error_data))
end_ts = time.time()
logging.info(
"calculate minute v1: {0} over, it takes: {1} seconds".
format(calculate_minute_item, int(end_ts - start_ts)))
def end_device_audio_record(self, end_ts):
device_id_list = list(self.device_audio_record.keys())
jobs = {}
job_index = 0
for device_id in device_id_list:
device_info = self.device_audio_record[device_id]
last_action = device_info.get("action")
last_update_ts = device_info.get("update_ts")
last_start_type = device_info.get("start_type")
last_start_ts = device_info.get("start_ts")
last_phid = device_info.get("phid")
last_live = device_info.get("live")
last_pid = device_info.get("pid")
last_tid = device_info.get("tid")
last_netstat = device_info.get("netstat")
last_start_pos = device_info.get("start_pos")
last_end_pos = device_info.get("end_pos")
diff_ts = int((end_ts - last_update_ts).total_seconds())
if last_action == "pause":
last_end_ts = last_update_ts
last_end_type = "pause"
else:
if diff_ts >= 40:
last_end_ts = last_update_ts + timedelta(seconds=30)
else:
last_end_ts = end_ts
last_end_type = "stop"
if last_end_ts is not None and last_end_ts < last_start_ts:
last_end_ts = last_start_ts + timedelta(seconds=30)
jobs[job_index] = [
last_start_type, last_end_type,
convert_into_utc_time(last_end_ts)
if last_end_ts is not None else None, last_live, last_pid,
last_tid, last_netstat, last_start_pos, last_end_pos,
convert_into_utc_time(
last_start_ts.replace(second=0, minute=0,
hour=0)), device_id,
convert_into_utc_time(last_start_ts), last_phid
]
job_index += 1
del self.device_audio_record[device_id]
if len(jobs) > 0:
self.cassandra_connector.do_futures("update_device_audio_data",
jobs)
def check_device_audio_record(self, check_ts):
device_id_list = list(self.device_audio_record.keys())
jobs = {}
job_index = 0
for device_id in device_id_list:
device_info = self.device_audio_record[device_id]
last_action = device_info.get("action")
last_update_ts = device_info.get("update_ts")
last_start_type = device_info.get("start_type")
last_start_ts = device_info.get("start_ts")
last_phid = device_info.get("phid")
last_live = device_info.get("live")
last_pid = device_info.get("pid")
last_tid = device_info.get("tid")
last_netstat = device_info.get("netstat")
last_start_pos = device_info.get("start_pos")
last_end_pos = device_info.get("end_pos")
diff_ts = int((check_ts - last_update_ts).total_seconds())
if diff_ts >= 40:
if last_action == "pause":
last_end_ts = last_update_ts
last_end_type = "pause"
else:
last_end_ts = last_update_ts + timedelta(seconds=30)
last_end_type = "stop"
if last_end_ts is not None and last_end_ts < last_start_ts:
last_end_ts = last_start_ts + timedelta(seconds=30)
jobs[job_index] = [
last_start_type, last_end_type,
convert_into_utc_time(last_end_ts)
if last_end_ts is not None else None, last_live, last_pid,
last_tid, last_netstat, last_start_pos, last_end_pos,
convert_into_utc_time(
last_start_ts.replace(second=0, minute=0,
hour=0)), device_id,
convert_into_utc_time(last_start_ts), last_phid
]
job_index += 1
del self.device_audio_record[device_id]
if len(jobs) > 0:
self.cassandra_connector.do_futures("update_device_audio_data",
jobs)
def get_logs_from_queue(self):
for data_key in self.data_record.keys():
self.period_logs_info[data_key] = self.data_record.get(data_key)
while not self.stop_flag.get() == 1:
try:
if not self.audio_queue.empty():
try:
now_ts = datetime.now().replace(microsecond=0)
message = self.audio_queue.get()
log_info = json.loads(message.decode("utf-8"),
object_hook=date_decoder)
log_time = log_info.get("log_time")
log_minute = log_time.replace(second=0)
log_end_ts = log_minute + timedelta(
minutes=1) + timedelta(seconds=self.delay_value)
# """离线计算"""
# if log_minute not in self.period_logs_info:
# self.period_logs_info[log_minute] = {"log_list": [], "end_ts": log_end_ts}
#
# self.period_logs_info[log_minute]["log_list"].append(log_info)
"""实时计算"""
if now_ts <= log_end_ts:
if log_minute not in self.period_logs_info:
self.period_logs_info[log_minute] = {
"log_list": [],
"end_ts": log_end_ts
}
self.period_logs_info[log_minute][
"log_list"].append(log_info)
except:
logging.error(message)
else:
logging.info("audio queue is empty...")
time.sleep(1)
except:
logging.error("audio queue is broken!!!")
break