Пример #1
0
def ff_log_parser(log_file, visit_info):
    events = []
    fp_events = []
    for line in gen_cat_file(log_file):
        # print line
        if line.startswith("FPD"):
            line = line.replace("FPD\t", "").rstrip()
            be = cm.BrowserEvent()
            log_items = line.split("\t")
            if len(log_items) == 5:
                be.initiator, be.event_type, be.js_file,\
                    be.js_line, be.log_text = log_items
                be.url = visit_info.url
                be.rank = visit_info.rank
                events.append(be)
            elif len(log_items) == 4 and log_items[1] in cm.EVENTS_WITH_4_ARGS:
                be.initiator, cm.event_type, be.js_file,\
                    be.js_line = log_items
                # filter out screenshots taken by browser for page thumbnails
                if not be.js_line.startswith('resource'):
                    events.append(be)
            else:
                print "Hmm WAT!:", log_items

    canvas_fp_urls = check_canvas_rw_access(events)
    for canvas_fp_url in canvas_fp_urls:
        for event in events:
            # we only add events from this URL
            if event.js_file == canvas_fp_url:
                fp_events.append(event)
    return fp_events, events
Пример #2
0
def ff_log_parser(log_file, visit_info):
    events = []
    fp_events = []
    for line in gen_cat_file(log_file):
        # print line
        if line.startswith("FPD"):
            line = line.replace("FPD\t", "").rstrip()
            be = cm.BrowserEvent()
            log_items = line.split("\t")
            if len(log_items) == 5:
                be.initiator, be.event_type, be.js_file,\
                    be.js_line, be.log_text = log_items
                be.url = visit_info.url
                be.rank = visit_info.rank
                events.append(be)
            elif len(log_items) == 4 and log_items[1] in cm.EVENTS_WITH_4_ARGS:
                be.initiator, cm.event_type, be.js_file,\
                    be.js_line = log_items
                # filter out screenshots taken by browser for page thumbnails
                if not be.js_line.startswith('resource'):
                    events.append(be)
            else:
                print "Hmm WAT!:", log_items

    canvas_fp_urls = check_canvas_rw_access(events)
    for canvas_fp_url in canvas_fp_urls:
        for event in events:
            # we only add events from this URL
            if event.js_file == canvas_fp_url:
                fp_events.append(event)
    return fp_events, events
Пример #3
0
def parse_strace_logs(visit_info, test_lso=None):
    events = []
    lines_seen = Set()
    # cm.print_log(visit_info, visit_info.sys_log, cm.LOG_DEBUG)
    for line in fu.gen_cat_file(visit_info.sys_log):
        if (".macromedia/Flash_Player/#SharedObjects" not in line or
                (line in lines_seen)):
            continue

        lines_seen.add(line)
        try:
            pieces = line.split("#SharedObjects/")[1].split("/")
            lso_file, mode_str = pieces[-1].split("\"")
            if "O_RDWR" in mode_str:
                mode = cm.ACCESS_MODE_READ_WRITE
            else:
                mode = cm.ACCESS_MODE_READ_ONLY
            domain = pieces[1]
            filename = line.split("\"")[1]
            file_ext = filename.rsplit(".", 1)[-1]
            # We observe .sxx extension for LSOs instead of .sol in strace logs
            # We recover the real filename by replacing
            # [pid 26407] open("/home/xyz/.macromedia/Flash_Player/#SharedObjects/GWZSHMBL/securehomes.esat.kuleuven.be/FlashCookie.sxx", O_RDWR|O_CREAT|O_APPEND, 0666) = 18  # noqa
            # [pid 26407] write(18, "\0\277\0\0\0-TCSO\0\4\0\0\0\0\0\vFlashCookie\0\0\0\3\21test_key\6\rjb0uf9\0", 51) = 51  # noqa
            # the only file in SharedObjects/GWZSHMBL/securehomes.esat.kuleuven.be would be a .sol file, which is not in the strace logs if it's created on this visit.  # noqa
            if file_ext == "sxx":
                filename = filename.replace(".sxx", ".sol")
            if test_lso is not None:  # just to simplify tests
                filename = test_lso  # override the lso filename
            if not filename.endswith(".sol"):
                print "Unexpected LSO file extension", filename, file_ext, line
                continue
            if not os.path.isfile(filename):
                # Happens when the (temp) LSO is removed before the visit ends.
                print "Cannot find LSO file", filename, file_ext, line
                continue
        except ValueError:
            continue
        # TODO: move below code to a separate function
        try:
            lso_dict = sol.load(filename)
            for k, v in lso_dict.iteritems():
                be = cm.BrowserEvent()
                be.event_type = cm.EVENT_FLASH_LSO
                be.js_file = lso_file
                be.initiator = domain
                be.mode = mode  # this doesn't seem to work
                be.cookie_path = filename.split("#SharedObjects/")[1]
                be.key = unicode(k)
                try:
                    be.log_text = unicode(v)
                except UnicodeDecodeError:  # obj is byte string
                    ascii_text = str(v).encode('string_escape')
                    be.log_text = unicode(ascii_text)
                events.append(be)
        except Exception as exc:
            cm.print_log(visit_info, "Error parsing LSO %s %s" %
                         (filename, exc))
    return events
Пример #4
0
def parse_strace_logs(visit_info, test_lso=None):
    events = []
    lines_seen = Set()
    # cm.print_log(visit_info, visit_info.sys_log, cm.LOG_DEBUG)
    for line in fu.gen_cat_file(visit_info.sys_log):
        if (".macromedia/Flash_Player/#SharedObjects" not in line
                or (line in lines_seen)):
            continue

        lines_seen.add(line)
        try:
            pieces = line.split("#SharedObjects/")[1].split("/")
            lso_file, mode_str = pieces[-1].split("\"")
            if "O_RDWR" in mode_str:
                mode = cm.ACCESS_MODE_READ_WRITE
            else:
                mode = cm.ACCESS_MODE_READ_ONLY
            domain = pieces[1]
            filename = line.split("\"")[1]
            file_ext = filename.rsplit(".", 1)[-1]
            # We observe .sxx extension for LSOs instead of .sol in strace logs
            # We recover the real filename by replacing
            # [pid 26407] open("/home/xyz/.macromedia/Flash_Player/#SharedObjects/GWZSHMBL/securehomes.esat.kuleuven.be/FlashCookie.sxx", O_RDWR|O_CREAT|O_APPEND, 0666) = 18  # noqa
            # [pid 26407] write(18, "\0\277\0\0\0-TCSO\0\4\0\0\0\0\0\vFlashCookie\0\0\0\3\21test_key\6\rjb0uf9\0", 51) = 51  # noqa
            # the only file in SharedObjects/GWZSHMBL/securehomes.esat.kuleuven.be would be a .sol file, which is not in the strace logs if it's created on this visit.  # noqa
            if file_ext == "sxx":
                filename = filename.replace(".sxx", ".sol")
            if test_lso is not None:  # just to simplify tests
                filename = test_lso  # override the lso filename
            if not filename.endswith(".sol"):
                print "Unexpected LSO file extension", filename, file_ext, line
                continue
            if not os.path.isfile(filename):
                # Happens when the (temp) LSO is removed before the visit ends.
                print "Cannot find LSO file", filename, file_ext, line
                continue
        except ValueError:
            continue
        # TODO: move below code to a separate function
        try:
            lso_dict = sol.load(filename)
            for k, v in lso_dict.iteritems():
                be = cm.BrowserEvent()
                be.event_type = cm.EVENT_FLASH_LSO
                be.js_file = lso_file
                be.initiator = domain
                be.mode = mode  # this doesn't seem to work
                be.cookie_path = filename.split("#SharedObjects/")[1]
                be.key = unicode(k)
                try:
                    be.log_text = unicode(v)
                except UnicodeDecodeError:  # obj is byte string
                    ascii_text = str(v).encode('string_escape')
                    be.log_text = unicode(ascii_text)
                events.append(be)
        except Exception as exc:
            cm.print_log(visit_info,
                         "Error parsing LSO %s %s" % (filename, exc))
    return events
Пример #5
0
 def test_gen_cat_file(self):
     result = fu.gen_cat_file(os.path.join(REGEXP_FILES_DIR,
                                           'regexp.txt'))
     self.failUnless(inspect.isgenerator(result))
     self.assertEqual(6, len(list(result)))