def __init__(self):
Monitor.__init__(self)
self.is_android = False
self.service_context = None
self.__test_android()
# self.ws_dissector_path = "ws_dissector"
self.ws_dissector_path = None
self.libs_path = None
if self.is_android:
libs_path = self.__get_libs_path()
ws_dissector_path = os.path.join(libs_path, "android_pie_ws_dissector")
self.libs_path = libs_path
self.ws_dissector_path = ws_dissector_path
prefs = {
"ws_dissect_executable_path": os.path.join(
libs_path,
"android_pie_ws_dissector"),
"libwireshark_path": libs_path}
else:
prefs = {}
print prefs
DMLogPacket.init(prefs)
self._type_names = []
def __init__(self, prefs={}):
Monitor.__init__(self)
self._input_dir = os.path.join(get_cache_dir(),
"mi2log") # ??? getCacheDir()
self._input_dir = "/sdcard/mtklog"
self._type_names = []
self._filename_sort_key = None
self._read_latency = []
libs_path = os.path.join(get_files_dir(), "data")
ws_dissector_path = os.path.join(libs_path, "android_pie_ws_dissector")
self.libs_path = libs_path
self.ws_dissector_path = ws_dissector_path
self._executable_path = os.path.join(self.libs_path,
"diag_revealer_mtk")
self._log_path = os.path.join(self._input_dir, "mdlog1")
self._filter_src_path = os.path.join(self.libs_path, "MTK-filter")
self._filter_dest_path = os.path.join(self._input_dir, "mdlog1_config")
prefs = {
"ws_dissect_executable_path":
os.path.join(libs_path, "android_pie_ws_dissector"),
"libwireshark_path":
libs_path
}
DMLogPacket.init(prefs)
def __init__(self):
"""
Configure this class with user preferences.
This method should be called before any actual decoding.
"""
Monitor.__init__(self)
self._fifo_path = self.TMP_FIFO_FILE
self._input_dir = os.path.join(get_cache_dir(), "mi2log")
self._log_cut_size = 0.5 # change size to 1.0 M
# self._skip_decoding = False
self._type_names = []
self._last_diag_revealer_ts = None
"""
Exec/lib initialization path
"""
# self._executable_path = "/system/bin/diag_revealer"
# prefs={"ws_dissect_executable_path": "/system/bin/android_pie_ws_dissector",
# "libwireshark_path": "/system/lib"}
libs_path = os.path.join(get_files_dir(), "data")
# libs_path = "./data"
self._executable_path = os.path.join(libs_path, "diag_revealer")
prefs = {
"ws_dissect_executable_path":
os.path.join(libs_path, "android_pie_ws_dissector"),
"libwireshark_path":
libs_path
}
DMLogPacket.init(prefs) # Initialize Wireshark dissector
def run(self):
"""
Start monitoring the mobile network. This is usually the entrance of monitoring and analysis.
"""
# fd = open('./Diag.cfg','wb')
# dm_collector_c.generate_diag_cfg(fd, self._type_names)
# fd.close()
self.log_info("Running Offline replayer")
try:
self.broadcast_info('STARTED',{})
log_list = []
if os.path.isfile(self._input_path):
log_list = [self._input_path]
elif os.path.isdir(self._input_path):
for file in os.listdir(self._input_path):
if file.endswith(".muxraw"):
log_list.append(os.path.join(self._input_path, file))
else:
self.log_debug("No files???")
return
log_list.sort() # Hidden assumption: logs follow the diag_log_TIMSTAMP_XXX format
# mtk_log_parser.ws_dissector_proc_start(self.ws_dissector_path, self.libs_path)
for file in log_list:
self.log_info("Loading " + file)
self._input_file = open(file, "rb")
dm_collector_c.reset()
while True:
s = self._input_file.read()
if not s:
break
decoded = mtk_log_parser.feed_binary(s) # self for debug
# decoded = mtk_log_parser.receive_log_packet(self._skip_decoding,
# True # include_timestamp
# )
######################################
if decoded != []:
try:
# # packet = DMLogPacket(decoded)
# packet = DMLogPacket(decoded[0])
# d = packet.decode()
# # print d["type_id"], d["timestamp"]
# # xml = packet.decode_xml()
# # print xml
# # print ""
# # Send event to analyzers
# event = Event( timeit.default_timer(),
# d["type_id"],
# packet)
# self.send(event)
##############################################
for msg in decoded:
typeid, rawid, msgstr = mtk_log_parser.decode(self, msg) #self for debug
if typeid == "":
continue
packet = DMLogPacket([("log_msg_len", len(msg), ""),('type_id', typeid, ''),('timestamp', datetime.datetime.now(), ''),("Msg", msgstr, "msg")]) # ("Msg", msgstr, "raw_msg/" + rawid)])
# print "DMLogPacket decoded[0]:",str([typeid])
event = Event( timeit.default_timer(), typeid, packet)
self.send(event)
##############################################
except FormatError as e:
print "FormatError: ", e # skip this packet
self._input_file.close()
except Exception as e:
import traceback
print str(traceback.format_exc())
def run(self):
"""
Start monitoring the mobile network. This is usually the entrance of monitoring and analysis.
"""
# fd = open('./Diag.cfg','wb')
# dm_collector_c.generate_diag_cfg(fd, self._type_names)
# fd.close()
try:
self.broadcast_info('STARTED', {})
log_list = []
if os.path.isfile(self._input_path):
log_list = [self._input_path]
elif os.path.isdir(self._input_path):
for file in os.listdir(self._input_path):
if file.endswith(".mi2log") or file.endswith(".qmdl"):
# log_list.append(self._input_path+"/"+file)
log_list.append(os.path.join(self._input_path, file))
else:
return
log_list.sort(
) # Hidden assumption: logs follow the diag_log_TIMSTAMP_XXX format
for file in log_list:
self.log_info("Loading " + file)
self._input_file = open(file, "rb")
dm_collector_c.reset()
while True:
s = self._input_file.read(64)
if not s: # EOF encountered
break
dm_collector_c.feed_binary(s)
# decoded = dm_collector_c.receive_log_packet()
decoded = dm_collector_c.receive_log_packet(
self._skip_decoding,
True, # include_timestamp
)
if decoded:
try:
# packet = DMLogPacket(decoded)
packet = DMLogPacket(decoded[0])
# print "DMLogPacket decoded[0]:",str(decoded[0])
d = packet.decode()
# print d["type_id"], d["timestamp"]
# xml = packet.decode_xml()
# print xml
# print ""
# Send event to analyzers
if d["type_id"] in self._type_names:
event = Event(timeit.default_timer(),
d["type_id"], packet)
self.send(event)
except FormatError as e:
# skip this packet
print "FormatError: ", e
self._input_file.close()
except Exception as e:
import traceback
sys.exit(str(traceback.format_exc()))
def run(self):
"""
Start monitoring the mobile network. This is usually the entrance of monitoring and analysis.
"""
# fd = open('./Diag.cfg','wb')
# dm_collector_c.generate_diag_cfg(fd, self._type_names)
# fd.close()
try:
self.broadcast_info('STARTED', {})
self.log_info('STARTED: ' + str(time.time()))
log_list = []
if os.path.isfile(self._input_path):
log_list = [self._input_path]
elif os.path.isdir(self._input_path):
for file in os.listdir(self._input_path):
if file.endswith(".mi2log") or file.endswith(".qmdl"):
# log_list.append(self._input_path+"/"+file)
log_list.append(os.path.join(self._input_path, file))
else:
return
log_list.sort(
) # Hidden assumption: logs follow the diag_log_TIMSTAMP_XXX format
decoding_inter = 0
sending_inter = 0
for file in log_list:
self.log_info("Loading " + file)
self.log_info('Loading: ' + str(time.time()))
self._input_file = open(file, "rb")
dm_collector_c.reset()
while True:
s = self._input_file.read(64)
if not s: # EOF encountered
break
dm_collector_c.feed_binary(s)
decoded = dm_collector_c.receive_log_packet(
self._skip_decoding,
True, # include_timestamp
)
if decoded:
try:
before_decode_time = time.time()
# self.log_info('Before decoding: ' + str(time.time()))
packet = DMLogPacket(decoded[0])
type_id = packet.get_type_id()
after_decode_time = time.time()
decoding_inter += after_decode_time - before_decode_time
if type_id in self._type_names:
event = Event(timeit.default_timer(), type_id,
packet)
self.send(event)
after_sending_time = time.time()
sending_inter += after_sending_time - after_decode_time
# pself.log_info('After sending event: ' + str(time.time()))
except FormatError as e:
# skip this packet
print "FormatError: ", e
self.log_info('Decoding_inter: ' + str(decoding_inter))
self.log_info('sending_inter: ' + str(sending_inter))
self._input_file.close()
except Exception as e:
import traceback
sys.exit(str(traceback.format_exc()))
# sys.exit(e)
event = Event(timeit.default_timer(), 'Monitor.STOP', None)
self.send(event)
self.log_info("Offline replay is completed.")
def _read_muxraw(self, f, cur_pos):
"""
Return final position.
"""
# mtk_log_parser.ws_dissector_proc_start(self.ws_dissector_path, self.libs_path)
BLOCK_SIZE = 128
f.seek(cur_pos, 0)
while True:
t1 = timeit.default_timer()
# s = f.read(BLOCK_SIZE)
s = f.read()
if not s: # EOF encountered
break
t2 = timeit.default_timer()
self._read_latency.append((float(t2 - t1), len(s)))
# dm_collector_c.feed_binary(s)
# decoded = dm_collector_c.receive_log_packet(self._skip_decoding,
# True, # include_timestamp
# )
######################################
# self.log_debug('before feed_binary: '+"\\x".join("{:02x}".format(ord(c)) for c in s))
decoded = mtk_log_parser.feed_binary(s) # self for debug
# decoded = mtk_log_parser.receive_log_packet(self._skip_decoding,
# True # include_timestamp
# )
######################################
if decoded != []:
try:
# # packet = DMLogPacket(decoded)
# packet = DMLogPacket(decoded[0])
# d = packet.decode()
# # print d["type_id"], d["timestamp"]
# # xml = packet.decode_xml()
# # print xml
# # print ""
# # Send event to analyzers
# event = Event( timeit.default_timer(),
# d["type_id"],
# packet)
# self.send(event)
##############################################
for msg in decoded:
typeid, rawid, msgstr = mtk_log_parser.decode(
self, msg) #self for debug
if typeid == "":
continue
packet = DMLogPacket([("log_msg_len", len(msg), ""),
('type_id', typeid, ''),
('timestamp',
datetime.datetime.now(), ''),
("Msg", msgstr, "msg")])
# ("Msg", msgstr, "raw_msg/" + rawid)]) #TODO: optimize parsing speed
event = Event(timeit.default_timer(), typeid, packet)
self.send(event)
##############################################
except FormatError as e:
self.log_error("FormatError: %s" %
str(e)) # skip this packet
return f.tell()
def run(self):
"""
Start monitoring the mobile network. This is usually the entrance of monitoring and analysis.
This function does NOT return or raise any exception.
"""
# Stop running loggers
# self._stop_collection()
self.broadcast_info('STARTED', {})
generate_diag_cfg = True
if not self._type_names:
raise RuntimeError(
"Log type not specified. Please specify the log types with enable_log()."
)
# if os.path.exists(os.path.join(self.DIAG_CFG_DIR, "Diag.cfg")):
# generate_diag_cfg = False
# # print "AndroidDevDiagMonitor: existing Diag.cfg file will be used."
# else:
# raise RuntimeError("Log type not specified. Please call enable_log() first.")
try:
if generate_diag_cfg:
if not os.path.exists(self.DIAG_CFG_DIR):
os.makedirs(self.DIAG_CFG_DIR)
# fd = open(os.path.join(self.DIAG_CFG_DIR, "Diag.cfg"), "w+b")
# Overwrite Diag.cfg, not append it
fd = open(os.path.join(self.DIAG_CFG_DIR, "Diag.cfg"), "wb")
dm_collector_c.generate_diag_cfg(fd, self._type_names)
fd.close()
self._mkfifo(self._fifo_path)
# Launch diag_revealer, and protection daemon
self._start_diag_revealer()
self.diag_revealer_daemon = threading.Thread(
target=self._protect_diag_revealer)
self.diag_revealer_daemon.start()
# fifo = os.open(self._fifo_path, os.O_RDONLY | os.O_NONBLOCK)
# Blocking mode: save CPU
fifo = os.open(self._fifo_path, os.O_RDONLY)
# Read log packets from diag_revealer
chproc = ChronicleProcessor()
while True:
try:
# self.log_info("Before os.read(fifo, self.BLOCK_SIZE)")
s = os.read(fifo, self.BLOCK_SIZE)
# self.log_info("After os.read(fifo, self.BLOCK_SIZE)")
except OSError as err:
if err.errno == errno.EAGAIN or err.errno == errno.EWOULDBLOCK:
# self.log_info("err.errno="+str(err.errno))
s = None
else:
raise err # something else has happened -- better reraise
while s: # preprocess metadata
# self.log_info("Before chproc.process: %s" % s)
ret_msg_type, ret_ts, ret_payload, ret_filename, remain = chproc.process(
s)
# self.log_info("After chproc.process(s)")
if ret_msg_type == ChronicleProcessor.TYPE_LOG:
if ret_ts:
self._last_diag_revealer_ts = ret_ts
if ret_payload:
dm_collector_c.feed_binary(ret_payload)
elif ret_msg_type == ChronicleProcessor.TYPE_START_LOG_FILE:
if ret_filename:
pass
# print "Start of %s" % ret_filename
elif ret_msg_type == ChronicleProcessor.TYPE_END_LOG_FILE:
if ret_filename:
# res_dict = {'filename':ret_filename}
# msg = ("new_diag_log",res_dict,"dict")
msg = ('filename', ret_filename, "")
# print "End of %s" % ret_filename
event = Event(timeit.default_timer(),
"new_diag_log", DMLogPacket([msg]))
# ret_filename)
self.send(event)
elif ret_msg_type is not None:
raise RuntimeError("Unknown ret msg type: %s" %
str(ret_msg_type))
s = remain
result = dm_collector_c.receive_log_packet(
self._skip_decoding,
True, # include_timestamp
)
if result: # result = (decoded, posix_timestamp)
try:
packet = DMLogPacket(result[0])
d = packet.decode()
# print d["type_id"], d["timestamp"], result[1]
# xml = packet.decode_xml()
# print xml
# print ""
# Send event to analyzers
event = Event(result[1], d["type_id"], packet)
self.send(event)
except FormatError as e:
# skip this packet
print "FormatError: ", e
except (KeyboardInterrupt, RuntimeError) as e:
os.close(fifo)
# proc.terminate()
self._stop_collection()
packet = DMLogPacket([])
event = Event(timeit.default_timer(), "sys_shutdown", packet)
self.send(event)
import traceback
sys.exit(str(traceback.format_exc()))
# sys.exit(e)
except Exception as e:
os.close(fifo)
# proc.terminate()
self._stop_collection()
packet = DMLogPacket([])
event = Event(timeit.default_timer(), "sys_shutdown", packet)
self.send(event)
import traceback
sys.exit(str(traceback.format_exc()))