def add_remove_clickable_tags(self): return dom = ''' <html><body> <form data-prop="formNode" class="sup-form sup-account-form"> <p> <input placeholder="Your name" data-prop="nameNode" data-event="input:onInfoInput" class="sup-info-name" data-l10n-id="setup-info-name" x-inputmode="verbatim" inputmode="verbatim" dir="auto" required="" type="text"> <button type="reset">reset1</button> </p> <p> <input placeholder="*****@*****.**" data-prop="emailNode" data-event="input:onInfoInput" class="sup-info-email" data-l10n-id="setup-info-email" dir="auto" required="" type="email"> <button type="reset">reset2</button> </p> <p> <button data-prop="nextButton" data-event="click:onNext" class="sup-info-next-btn recommend" disabled=""> <span data-l10n-id="setup-info-next">Next</span> </button> <button data-prop="manualConfig" data-event="click:onClickManualConfig" class="sup-manual-config-btn" disabled="" data-l10n-id="setup-manual-config2">Manual setup</button> </p> </form> </body></html> ''' # Tag('a'), Tag('button'), Tag('input', {'type': 'submit'}), Tag('input', {'type': 'button'}) self.assertEqual(len(DomAnalyzer.get_clickable_tags()), 4) DomAnalyzer.remove_clickable_tags(Tag('a')) DomAnalyzer.remove_clickable_tags(Tag('button')) DomAnalyzer.remove_clickable_tags(Tag('button')) DomAnalyzer.remove_clickable_tags(Tag('input', {'type': 'submit'})) DomAnalyzer.remove_clickable_tags(Tag('input', {'type': 'button'})) self.assertEqual(len(DomAnalyzer.get_clickable_tags()), 0) DomAnalyzer.add_clickable_tags(Tag('button', {'type': 'reset'})) self.assertEqual(len(DomAnalyzer.get_clickables(dom)), 2)
def get_clickable(self): dom = ''' <html><body> <form data-prop="formNode" class="sup-form sup-account-form"> <p> <input placeholder="Your name" data-prop="nameNode" data-event="input:onInfoInput" class="sup-info-name" data-l10n-id="setup-info-name" x-inputmode="verbatim" inputmode="verbatim" dir="auto" required="" type="text"> <button type="reset"></button> </p> <p> <input placeholder="*****@*****.**" data-prop="emailNode" data-event="input:onInfoInput" class="sup-info-email" data-l10n-id="setup-info-email" dir="auto" required="" type="email"> <button type="reset"></button> </p> <p> <button data-prop="nextButton" data-event="click:onNext" class="sup-info-next-btn recommend" disabled=""> <span data-l10n-id="setup-info-next">Next</span> </button> <button data-prop="manualConfig" data-event="click:onClickManualConfig" class="sup-manual-config-btn" disabled="" data-l10n-id="setup-manual-config2">Manual setup</button> </p> </form> </body></html> ''' clickables = DomAnalyzer.get_clickables(dom) self.assertEqual(len(clickables), 4) self.assertEqual(len(clickables[3].get_forms()), 1) self.assertEqual(len(clickables[3].get_forms()[0].get_inputs()), 2) self.assertEqual(clickables[3].get_forms()[0].get_inputs()[1].get_xpath(), '//html/body/form[1]/p[2]/input[1]') self.assertEqual(clickables[3].get_forms()[0].get_inputs()[1].get_type(), 'email') '''
def add_new_events(self, state, prev_state, depth): for clickables, iframe_key in DomAnalyzer.get_clickables(state, prev_state if prev_state else None): for clickable in clickables: self.crawler.action_events.append( { 'state' : state, 'action' : { 'clickable':clickable, 'iframe_key':iframe_key }, 'depth' : depth, } )
def crawl(self, depth, prev_state=None): if depth <= self.configuration.get_max_depth(): cs = self.automata.get_current_state() if not self.violate_invariant(cs.get_dom(), cs.get_id()): candidate_clickables = DomAnalyzer.get_clickables(cs.get_dom(), prev_state.get_dom() if prev_state else None) self.num_clickables['unexamined'] += len(candidate_clickables) for clickable in candidate_clickables: # prefetch image of the clickable time.sleep(0.2) # time for correctly fetching image img_name = cs.get_id() + '-' + clickable.get_id() + '.png' img_data = self.executor.get_screenshot(clickable) # fire the clickable logger.debug('Fire event in state %s', cs.get_id()) self.executor.empty_form(clickable) self.executor.fill_form(clickable) ft = FireEventThread(self.executor, clickable) ft.start() ft.join(self.configuration.get_sleep_time()*2) # time out after sleep_time*2 seconds if ft.is_alive(): # timed out logger.error('No response while firing an event. Execution sequences:') self.exe_stack.append(clickable) # add the clickable triggering No Response for c in self.exe_stack: logger.error(c) logger.error('Total clickables found: %d (true: %d, false: %d, unexamined: %d)', self.num_clickables['unexamined'] + self.num_clickables['true'] + self.num_clickables['false'], self.num_clickables['true'], self.num_clickables['false'], self.num_clickables['unexamined'] ) logger.error('Program terminated.') sys.exit() time.sleep(self.configuration.get_sleep_time()) self.num_clickables['unexamined'] -= 1 new_dom = self.executor.get_source() if DomAnalyzer.is_equal(cs.get_dom(), new_dom): self.num_clickables['false'] += 1 else: self.num_clickables['true'] += 1 cs.add_clickable(clickable) self.exe_stack.append(clickable) self.save_screenshot(img_name, img_data, 'clickable') ns, is_newly_added = self.automata.add_state(State(new_dom)) self.automata.add_edge(cs, ns, clickable) if is_newly_added: self.save_screenshot(ns.get_id() + '.png', self.executor.get_screenshot(), 'state') self.save_dom(ns) self.automata.change_state(ns) self.crawl(depth+1, cs) self.exe_stack.pop(-1) self.automata.change_state(cs) self.backtrack(cs)
def add_new_events(self, state, prev_state, depth): candidate_clickables = [] for clickables, iframe_key in DomAnalyzer.get_clickables(state, prev_state if prev_state else None): for clickable in clickables: candidate_clickables.append( (clickable, iframe_key) ) if not candidate_clickables: return clickable, iframe_key = random.choice( candidate_clickables ) self.crawler.action_events.append( { 'state' : state, 'action' : { 'clickable':clickable, 'iframe_key':iframe_key }, 'depth' : depth, } ) print(state.get_id(),clickable.get_id(), clickable.get_xpath())
def crawl(self, depth, prev_state=None): if depth <= self.configuration.get_max_depth(): cs = self.automata.get_current_state() if not self.violate_invariant(cs.get_dom(), cs.get_id()): candidate_clickables = DomAnalyzer.get_clickables( cs.get_dom(), prev_state.get_dom() if prev_state else None) self.num_clickables['unexamined'] += len(candidate_clickables) for clickable in candidate_clickables: # prefetch image of the clickable time.sleep(0.2) # time for correctly fetching image img_name = cs.get_id() + '-' + clickable.get_id() + '.png' img_data = self.executor.get_screenshot(clickable) # fire the clickable logger.debug('Fire event in state %s', cs.get_id()) self.executor.empty_form(clickable) self.executor.fill_form(clickable) ft = FireEventThread(self.executor, clickable) ft.start() ft.join(self.configuration.get_sleep_time() * 2) # time out after sleep_time*2 seconds if ft.is_alive(): # timed out logger.error( 'No response while firing an event. Execution sequences:' ) self.exe_stack.append( clickable ) # add the clickable triggering No Response for c in self.exe_stack: logger.error(c) logger.error( 'Total clickables found: %d (true: %d, false: %d, unexamined: %d)', self.num_clickables['unexamined'] + self.num_clickables['true'] + self.num_clickables['false'], self.num_clickables['true'], self.num_clickables['false'], self.num_clickables['unexamined']) logger.error('Program terminated.') sys.exit() time.sleep(self.configuration.get_sleep_time()) self.num_clickables['unexamined'] -= 1 new_dom = self.executor.get_source() if DomAnalyzer.is_equal(cs.get_dom(), new_dom): self.num_clickables['false'] += 1 else: self.num_clickables['true'] += 1 cs.add_clickable(clickable) self.exe_stack.append(clickable) self.save_screenshot(img_name, img_data, 'clickable') ns, is_newly_added = self.automata.add_state( State(new_dom)) self.automata.add_edge(cs, ns, clickable) if is_newly_added: self.save_screenshot( ns.get_id() + '.png', self.executor.get_screenshot(), 'state') self.save_dom(ns) self.automata.change_state(ns) self.crawl(depth + 1, cs) self.exe_stack.pop(-1) self.automata.change_state(cs) self.backtrack(cs)