def SeleniumMain(web_submit_id, folderpath=None, dirname=None):
logging.info(" connect to mysql")
print("connect to sql")
databank = MysqlDataBank("localhost", "root", "", "test")
url, deep, time = databank.get_websubmit(web_submit_id)
logging.info(" setting config...")
print(" setting config...")
config = SeleniumConfiguration(Browser.PhantomJS, url, folderpath, dirname)
config.set_max_depth(deep)
config.set_max_time(int(time)*60)
config.set_simple_clickable_tags()
config.set_simple_inputs_tags()
config.set_simple_normalizers()
config.set_frame_tags(['iframe'])
logging.info(" setting executor...")
executor = SeleniumExecutor(config.get_browserID(), config.get_url())
logging.info(" setting crawler...")
automata = Automata()
crawler = SeleniumCrawler(config, executor, automata, databank)
logging.info(" crawler start run...")
automata = crawler.run()
crawler.close()
logging.info(" end! save automata...")
automata.save_automata(config)
automata.save_traces(config)
Visualizer.generate_html('web', os.path.join(config.get_path('root'), config.get_automata_fname()))
config.save_config('config.json')
def debugTestMain(folderpath, dirname):
logging.info(" setting config...")
config = SeleniumConfiguration(Browser.FireFox, "http://140.112.42.145:2000/demo/nothing/main.html")
config.set_max_depth(1)
config.set_max_length(5)
config.set_trace_amount(1)
config.set_max_states(100)
config.set_folderpath(folderpath)
config.set_dirname(dirname)
config.set_automata_fname('automata.json')
config.set_traces_fname('traces.json')
config.set_frame_tags(['iframe'])
config.set_dom_inside_iframe(True)
config.set_simple_clickable_tags()
config.set_simple_inputs_tags()
config.set_simple_normalizers()
logging.info(" setting executor...")
executor = SeleniumExecutor(config.get_browserID(), config.get_url())
logging.info(" setting crawler...")
automata = Automata(config)
databank = InlineDataBank("140.112.42.145:2000", "jeff", "zj4bj3jo37788", "test")
algorithm = MonkeyCrawler() #DFScrawler()
crawler = SeleniumCrawler(config, executor, automata, databank, algorithm)
logging.info(" crawler start run...")
crawler.run_algorithm()
logging.info(" end! save automata...")
algorithm.save_traces()
automata.save_automata(config.get_automata_fname())
Visualizer.generate_html('web', os.path.join(config.get_path('root'), config.get_automata_fname()))
config.save_config('config.json')
def SeleniumMutationTrace(folderpath, dirname, config_fname, traces_fname,
trace_id, method_id, modes):
logging.info(" loading config...")
config = load_config(config_fname)
config.set_folderpath(folderpath)
config.set_dirname(dirname)
config.set_mutation_trace(traces_fname, trace_id)
config.set_mutation_method(method_id)
config.set_mutation_modes(modes)
logging.info(" setting executor...")
executor = SeleniumExecutor(config.get_browserID(), config.get_url())
logging.info(" setting crawler...")
automata = Automata()
databank = MysqlDataBank("localhost", "jeff", "zj4bj3jo37788", "test")
crawler = SeleniumCrawler(config, executor, automata, databank)
logging.info(" crawler start run...")
crawler.run_mutant()
logging.info(" end! save automata...")
automata.save_traces(config)
automata.save_automata(config)
Visualizer.generate_html(
'web',
os.path.join(config.get_path('root'), config.get_automata_fname()))
def add_executor(self, argv):
self.other_executor = SeleniumExecutor(browserID, config.get_url())
class CBTMonkeyCrawler(AlgoCrawler):
def __init__(self):
self.trace_length_count = 0
self.traces = []
self.trace_history = {}
self.other_executor = None
def add_executor(self, argv):
self.other_executor = SeleniumExecutor(browserID, config.get_url())
def set_utility(self, crawler, configuration, executor, automata):
self.crawler = crawler
self.configuration = configuration
self.executor = executor
self.automata = automata
def prepare(self):
#executor
self.executor.start()
self.executor.goto_url()
self.other_executor.start()
self.other_executor.goto_url()
#initial state
initial_state = self.crawler.get_initail_state()
#save trace
self.trace_length_count = 0
self.trace_history = { 'states': [ initial_state ], 'edges': [] }
self.crawler.run_script_before_crawl(initial_state)
def add_new_events(self, state, prev_state, depth):
candidate_clickables = []
for clickables, iframe_key in DomAnalyzer.get_clickables(state, prev_state if prev_state else None):
for clickable in clickables:
candidate_clickables.append( (clickable, iframe_key) )
if not candidate_clickables:
return
clickable, iframe_key = random.choice( candidate_clickables )
print(state.get_id(),clickable.get_id(), clickable.get_xpath())
self.crawler.action_events.append( {
'state' : state,
'action' : { 'clickable':clickable, 'iframe_key':iframe_key },
'depth' : depth,
} )
def get_next_action(self, action_events):
event = action_events.pop()
return event
def change_state(self, state, action, depth):
logging.info('==========< BACKTRACK START >==========')
logging.info('==<BACKTRACK> depth %s -> backtrack to state %s',depth ,state.get_id() )
self.crawler.executor_backtrack(state, self.executor, self.other_executor)
logging.info('==========< BACKTRACK END >==========')
def trigger_action(self, state, new_edge, action, depth):
logging.info(' |depth:%s state:%s| fire element in iframe(%s)', depth, state.get_id(), action['iframe_key'])
self.trace_length_count += 1
self.crawler.make_value(new_edge)
self.executor.click_event_by_edge(new_edge)
self.other_executor.click_event_by_edge(new_edge)
def update_with_same_state(self, current_state, new_edge, action, depth, dom_list, url):
#save trace
self.trace_history['states'].append(current_state)
self.trace_history['edges'].append(new_edge)
#get new event again
if self.trace_length_count < self.configuration.get_max_length():
self.crawler.add_new_events(current_state, None, depth)
def update_with_out_of_domain(self, current_state, new_edge, action, depth, dom_list, url):
#save trace
self.trace_history['states'].append(current_state)
self.trace_history['edges'].append(new_edge)
# back if state out of domain
logging.info(' |depth:%s state:%s| out of domain: %s', depth, current_state.get_id(), url)
logging.info('==========< BACKTRACK START >==========')
logging.info('==<BACKTRACK> depth %s -> backtrack to state %s',depth ,current_state.get_id() )
self.crawler.executor_backtrack(current_state, self.executor, self.other_executor)
logging.info('==========< BACKTRACK END >==========')
#get new event again
if self.trace_length_count < self.configuration.get_max_length():
self.crawler.add_new_events(current_state, None, depth)
def update_with_new_state(self, current_state, new_state, new_edge, action, depth, dom_list, url):
#save trace
self.trace_history['states'].append(new_state)
self.trace_history['edges'].append(new_edge)
# automata save new state
logging.info(' |depth:%s state:%s| add new state %s of : %s', depth, current_state.get_id(), new_state.get_id(), url )
self.automata.save_state(self.executor, new_state, depth)
self.automata.save_state_shot(self.executor, new_state)
self.check_diff_browser()
if self.trace_length_count < self.configuration.get_max_length():
self.crawler.add_new_events(new_state, None, depth)
def update_with_old_state(self, current_state, new_state, new_edge, action, depth, dom_list, url):
#save trace
self.trace_history['states'].append(new_state)
self.trace_history['edges'].append(new_edge)
#check if old state have a shorter depth
if depth < new_state.get_depth():
new_state.set_depth(depth)
if depth > new_state.get_depth():
depth = new_state.get_depth()
if self.trace_length_count < self.configuration.get_max_length():
self.crawler.add_new_events(new_state, None, depth)
def save_traces(self):
self.automata.save_traces(self.traces)
def end(self):
self.traces.append( self.trace_history )
self.trace_length_count = 0
self.trace_history = {}
self.other_executor.close()
def check_diff_browser(self):
# 1. check executor other_executor is same state
# 2. if same , analsis
analysis_elements( self.executor )
analysis_elements( self.other_executor )
analysis_with_other_browser()
def analysis_elements(self, executor):
pass
def analysis_with_other_browser(self):
pass
def __init__(self):
self.trace_length_count = 0
self.traces = []
self.trace_history = {}
self.other_executor = SeleniumExecutor(Browser.Chrome, "http://140.112.42.145:2000/demo/nothing/main.html")