def download_page(self, page, training_stage): print_warning("[TRAINER] Opening page...") self.program.driver.load_url(page) self.program.driver.switch_to.window(self.program.driver.window_handles[0]) print_warning("[TRAINER] Downloading page...") pyautogui.hotkey('CTRL', '0') # TODO try to find a more efficient way to do it... sleep(16) '''if training_stage == TrainingStage.LOGIN_PAGE: pyautogui.hotkey('DOWN', 'ENTER') # Browser remembers about previously chosen action - save and not open else: pyautogui.hotkey('ENTER')''' # TODO FIX THIS if commands.getoutput("xset q | grep LED | awk '{print $10}' | cut -b 8") == 1: print "CAPS Enabled" pyautogui.press('CAPSLOCK') if commands.getoutput("xset q | grep LED | awk '{print $10}' | cut -b 8") == 1: print "CAPS STILL ENABLED" pyautogui.hotkey('ENTER') sleep(1) pyautogui.keyDown('shift') pyautogui.typewrite('7') pyautogui.keyUp('shift') pyautogui.typewrite("tmp") pyautogui.keyDown('shift') pyautogui.typewrite('7') pyautogui.keyUp('shift') pyautogui.typewrite("dump") pyautogui.hotkey('ENTER') sleep(2) print_successful_status("[TRAINER] ...done!")
def should_i_take_a_break(self): if random.random() > 0.97: print_warning( "[CRAWLER] Caronte needs to smoke a cigarette. Will work in a while." ) sleep(self.take_a_break()) else: print_warning("[CRAWLER] Not taking any break. Need to work!")
def dump_procedure(self): print_warning("[CRAWLER] Beginning dump procedure.") print_warning("[CRAWLER] Reaching forum landing page.") sleep(1) ''' Could be interesting thinking about how make this flexible to xpaths, ids and partial names. An option could we working with "generic finders" that specialize in front of the informations given from the database. The DB should then qualify the property exposed, so that the code can invoke the correct function. Moreover, a proper try except structure can be useful to address both the problems connected to a new page structure and for test purposes in training phase. ''' # TODO replace all load_url with click or different actions. load_url doesn't provide referral link. # Iterate forums cookie_value = "CARONTE" + str( int((datetime.datetime.today() - datetime.datetime(1970, 1, 1)).total_seconds())) for forum_xpath in self.website_dao.get_forum_xpaths_by_url( self.landing_page): wait_simple_pause() self.program.driver.load_url(self.landing_page) # Retrieving forum link for navigation forum_element = self.program.driver.find_element_by_xpath( forum_xpath) forum_link = forum_element.get_attribute("href") subforums = self.website_dao.get_subforum_xpaths_by_url_and_parent_xpath( self.landing_page, forum_xpath) # Subforums might be not present but directly posts if subforums is None or subforums[0] == '' or subforums[0] is None: wait_simple_pause() self.program.driver.click(forum_element) self.iterate_threads(forum_link) else: # Iterate subforums for subforum_xpath in subforums: wait_simple_pause() self.program.driver.load_url(forum_link) # Retrieving subforum link for navigation subforum_element = self.program.driver.find_element_by_xpath( subforum_xpath) subforum_link = subforum_element.get_attribute("href") wait_simple_pause() self.program.driver.click(subforum_element) self.iterate_threads(subforum_link)
def begin_training(self, forum_landing_page, login_page): # Check if the forum's structure is known if self.website_dao.get_website_id_by_url( forum_landing_page) is not None: print_successful_status( "[TRAINER] The forum is known. It is possible to proceed to the crawling." ) else: print_warning( "[TRAINER] The forum is not known. Proceeding to the learning phase." ) self.get_login_page_structure(login_page) # Executing login CrawlBootstrapper(self.program, self.db, login_page) self.get_relevant_forums(forum_landing_page) self.get_subforum_structure(forum_landing_page) self.get_thread_structure(forum_landing_page) return self.website_dao.get_website_id_by_url(forum_landing_page)
def remove_existing_js(self): print_warning("[TRAINER] Purging JS content from page...") with open("/tmp/dump.html") as oldfile, open("/tmp/no_js_dump.html", 'w') as newfile: script_open_tag = False for line in oldfile: pattern = re.compile("<script.*</script>") # This line is a an oneliner script? if pattern.match(line): # It is line = re.sub("<script.*/script>", "", line) # It is just an oneliner or there's something left? if line is not "": # There's something left. Lets write it on the file. newfile.write(line) # else, it is just script and i can skip it. else: # The line is not an oneliner script. pattern = re.compile("<script.*") # Begin of a multiliner? if pattern.match(line): # Yes it is script_open_tag = True line = re.sub("<script.*>", "", line) # It is just starting or there's also something before? if line is not "": # There's something left. Lets write it on the file. newfile.write(line) # else, it is just starting and i can skip it else: pattern = re.compile(".*</script>") # End of a multiliner? if pattern.match(line): script_open_tag = False line = re.sub("<script.*/script>", "", line) # There's something left? if line is not "": # Lets write it on the file. newfile.write(line) # Outside of a multiliner? elif not script_open_tag: newfile.write(line)
def to_work_or_not_to_work(self): if self.work_schedule.start_time_morning is not None and self.now < self.work_schedule.start_time_morning: # Too early to work, schedule morning self.schedule_job_and_close( self.work_schedule.start_time_morning.hour, self.work_schedule.start_time_morning.minute, False) else: if self.work_schedule.start_time_morning is not None and self.now < self.work_schedule.end_time_morning: # Morning work session self.setup_end_job_handler( (self.work_schedule.end_time_morning - self.now).seconds, WorkSession.MORNING) else: if self.work_schedule.start_time_afternoon is not None and self.now < self.work_schedule.start_time_afternoon: # Too early to work, schedule afternoon self.schedule_job_and_close( self.work_schedule.start_time_afternoon.hour, self.work_schedule.start_time_afternoon.minute, False) else: if self.work_schedule.start_time_afternoon is not None and self.now < self.work_schedule.end_time_afternoon: # Afternoon work session self.setup_end_job_handler( (self.work_schedule.end_time_afternoon - self.now).seconds, WorkSession.AFTERNOON) else: # Too early to work, schedule evening if self.work_schedule.start_time_evening is not None and self.now < self.work_schedule.start_time_evening: self.schedule_job_and_close( self.work_schedule.start_time_evening.hour, self.work_schedule.start_time_evening.minute, False) else: if self.work_schedule.start_time_evening is not None and self.now < self.work_schedule.end_time_evening: # Evening work session self.setup_end_job_handler( (self.work_schedule.end_time_evening - self.now).seconds, WorkSession.EVENING) else: # Too late to work tonight. Let's go with tomorrow. tomorrow_work_schedule = WorkSchedule( (self.weekday + 1) % 7) if 0 <= (self.weekday + 1) % 7 <= 4: # I'll work tomorrow afternoon, since is weekday while tomorrow_work_schedule.start_time_afternoon is None: tomorrow_work_schedule = WorkSchedule( self.weekday) self.schedule_job_and_close( tomorrow_work_schedule. start_time_afternoon.hour, tomorrow_work_schedule. start_time_afternoon.minute, True) else: # I'll work tomorrow morning, since is weekend while tomorrow_work_schedule.start_time_morning is None: tomorrow_work_schedule = WorkSchedule( self.weekday) self.schedule_job_and_close( tomorrow_work_schedule. start_time_morning.hour, tomorrow_work_schedule. start_time_morning.minute, True) sleep(1) # Yeah, just for waiting SnoozeAlarm to print its stdout... self.print_work_schedule() # Linux's cron doesn't allow expressing seconds. For this reason, an analysis could make noticeable that all the # sessions start are at the same amounts of seconds, in good network conditions. seconds_to_wait = random.randint(1, 60) print_warning("[BOOTSTRAP] Delaying start time of " + str(seconds_to_wait) + " seconds...") sleep(seconds_to_wait) self.setup_breaks_handler()
def print_work_schedule(self): print_warning("===================") print_warning("Time now here: " + str(get_localzone().localize(datetime.now()))) print_warning("Time now in Moscow: " + str(datetime.now(pytz.timezone('Europe/Moscow')))) print_warning("===================") print_warning("Work schedule (in Europe/Moscow timezone): ") print_warning("====> Start morning: " + str(self.work_schedule.start_time_morning) + "; duration: " + str(self.work_schedule.actual_duration_minutes_morning)) print_warning( "====> Start afternoon: " + str(self.work_schedule.start_time_afternoon) + "; duration: " + str(self.work_schedule.actual_duration_minutes_afternoon)) print_warning("====> Start evening: " + str(self.work_schedule.start_time_evening) + "; duration: " + str(self.work_schedule.actual_duration_minutes_evening)) print_warning("===================")
def inject_js_listeners(self, training_stage): print_warning("[TRAINER] Injecting JS content...") jquery_content = open(os.path.join(os.path.join(self.ROOT_DIR, "resources"), "jquery-3.3.1.min.js"), "r").read() with open("/tmp/no_js_dump.html") as oldfile, open("/tmp/clean_dump.html", 'w') as newfile: # Click-and-trigger-HTTP-request listeners + call on close if (training_stage == TrainingStage.FORUM) or (training_stage == TrainingStage.SUBFORUM) or ( training_stage == TrainingStage.THREAD_POOL) or (training_stage == TrainingStage.POST_POOL) or ( training_stage == TrainingStage.THREAD_NEXT_PAGE) or ( training_stage == TrainingStage.SUBFORUM_NEXT_PAGE) or (training_stage == TrainingStage.LOGIN_PAGE): for line in oldfile: # Pre-editing for correcting body tag if "<body " in line or "<BODY " in line or "<body>" in line or "<BODY>" in line: index = line.find('>') output_line = line[:index] + " onload=\"setupListeners()\"" + line[index:] line = output_line newfile.write(line) # After writing head tag, let's inject the scripts if "<head " in line or "<HEAD " in line or "<head>" in line or "<HEAD>" in line: # Beautified snippet below if training_stage == TrainingStage.LOGIN_PAGE: injected_content = '<script>function setupListeners(){var ' \ 'Anchors=document.getElementsByTagName("a");for(var ' \ 'i=0;i<Anchors.length;i+=1){Anchors[i].removeAttribute("href")}var ' \ 'forms=document.getElementsByTagName("form");for(var ' \ 'i=0;i<forms.length;i+=1){forms[i].removeAttribute("action");forms[' \ 'i].removeAttribute("method")}}</script><script>var username_collected' \ '=false;var is_next_button=false;document.onclick=function(event){ ' \ 'console.log(event); var target=\'target\'in event?event.target:event.' \ 'srcElement;console.log(target); if(' \ 'target.tagName===\'INPUT\' || target.tagName===\'BUTTON\' ){console.' \ 'log(target.tagName);console.log(\'My thoughts against JS are censored' \ '. Why? Because removing any of these 4 prints up there causes it to ' \ 'fail to send the last HTTP request.\');console.log(\'Please leave ' \ 'this piece of black magic where it is. I hate doing things in this ' \ 'way, but Im done debugging JS. Well never be friends.\'); var ' \ 'root=document.compatMode===\'CSS1Compat\'?document.documentElement' \ ':document.body;var path=getPathTo(target);if(!username_collected){' \ '$.post("http://*****:*****@id=\'"+element.id+"\']"}if(element===document.body){return ' \ 'element.tagName.toLowerCase()}var ix=0;var ' \ 'siblings=element.parentNode.childNodes;for(var ' \ 'i=0;i<siblings.length;i+=1){var sibling=siblings[i];if(' \ 'sibling===element){return getPathTo(' \ 'element.parentNode)+\'/\'+element.tagName.toLowerCase()+\'[\'+(' \ 'ix+1)+\']\'}if(sibling.nodeType===1&&sibling.tagName===element.' \ 'tagName){ix+=1}}};alert("Please click on username field.")</script>' else: injected_content = '<script>function setupListeners(){for(var e=document.' \ 'getElementsByTagName("a"),n=0;n<e.length;n++)e[n].removeAttribute' \ '("href"),e[n].addEventListener("click",function(e){name_found=' \ 'getPathTo(this),$.post("http://' if training_stage == TrainingStage.FORUM: injected_content += 'localhost:8080/forum_name' if training_stage == TrainingStage.SUBFORUM: injected_content += 'localhost:8080/subforum_name' if training_stage == TrainingStage.SUBFORUM_NEXT_PAGE: injected_content += 'localhost:8080/subforum_next_page' if training_stage == TrainingStage.THREAD_POOL: injected_content += 'localhost:8080/thread_name' if training_stage == TrainingStage.THREAD_NEXT_PAGE: injected_content += 'localhost:8080/thread_next_page' injected_content += '",{name:name_found},function(e,n){console.log("error while posting' \ '..?")})},!1)}</script><script>' if training_stage == TrainingStage.FORUM: injected_content += 'alert("Click forums to crawl.' if training_stage == TrainingStage.SUBFORUM: injected_content += 'alert("Click subforums to crawl.' if training_stage == TrainingStage.SUBFORUM_NEXT_PAGE: injected_content += 'alert("Click the next page button.' if training_stage == TrainingStage.THREAD_POOL: injected_content += 'alert("Click exactly five thread titles.' if training_stage == TrainingStage.THREAD_NEXT_PAGE: injected_content += 'alert("Click the next page button.' injected_content += ' After you close the next page, put again Tor Browser on foreground,' \ ' if it isn\'t.");function ' \ 'getPathTo(element){if(element.id!==\'\'){return "//*[' \ '@id=\'"+element.id+"\']"}if(element===document.body){return ' \ 'element.tagName.toLowerCase()}var ix=0;var ' \ 'siblings=element.parentNode.childNodes;for(var ' \ 'i=0;i<siblings.length;i+=1){var sibling=siblings[i];if(' \ 'sibling===element){return getPathTo(' \ 'element.parentNode)+\'/\'+element.tagName.toLowerCase()+\'[\'+(' \ 'ix+1)+\']\'}if(sibling.nodeType===1&&sibling.tagName===element.' \ 'tagName){ix+=1}}}</script>' injected_content += '<script>window.addEventListener("beforeunload",function(e){$.get("' \ 'http://localhost:8080/closed",function(e,o){console.log("should really ' \ 'be triggered?")});return(e||window.event).returnValue="o/","o/"});' \ '</script><script>' + jquery_content + '</script>' newfile.write(injected_content) else: # Call on close for line in oldfile: newfile.write(line) if "<head " in line or "<HEAD " in line or "<head>" in line or "<HEAD>" in line: injected_content = "<script>window.addEventListener(\"beforeunload\",function(e){$.get(\"" \ "http://localhost:8080/closed\",function(e,o){console.log(\"should really " \ "be triggered?\")});return(e||window.event).returnValue=\"o/\",\"o/\"" \ "});</script><script>" + jquery_content + "</script>" newfile.write(injected_content) print_successful_status("[TRAINER] ...done!")
def run(self): print_warning("[CRAWLER] I will work for " + str(self.zzz/3600) + " hours and " + str((self.zzz % 3600)/60) + " minutes.") time.sleep(self.zzz) print_warning("[CRAWLER] Stop working for now. Scheduling next work session...") self.schedule_restart() os.kill(os.getpid(), signal.SIGTERM)
def iterate_threads(self, subforum_link): print_warning("[CRAWLER] Starting to read threads on page...") scanned_threads = [] first_page = True while True: # If all threads of the page have been scanned, move to next page thread_pool = self.get_forum_subelements(ForumField.THREAD_POOL, self.program.driver) if scanned_threads.__len__() == thread_pool.__len__(): scanned_threads = [] # Try to reach next page, if any. try: # vBulletin handles next pages differently compared to other forums. In particular, uses the same # tag and class to define the button of the next page, plus XPATH is not exploitable due the fact # it varies depending on thread length. if self.website_dao.get_is_vbullettin_by_website_id( self.website_id): prev_next_elements = self.get_forum_subelements( ForumField.SUBFORUM_NEXT_PAGE, self.program.driver) # If there is 4 navigation buttons (2 on top page, 2 on bottom page), or is the first page, # then there's a next page... try: if prev_next_elements.__len__() == 4 or ( prev_next_elements.__len__() == 2 and first_page): first_page = False subforum_link = prev_next_elements[ 1].get_attribute("href") if subforum_link is None: # Terrible, but should be worth it: if the current item has not the link, # it will likely be present in children. For this reason, I iterate on them until # I don't find the proper link. for child in prev_next_elements[ 1].find_elements_by_xpath(".//*"): subforum_link = child.get_attribute( "href") # Found it! if subforum_link != "" or not None: break # If I reach the end of for cycle, then there's a problem. raise NoSuchElementException self.program.driver.click( prev_next_elements[1]) # ...else, could be or a first or a last page. else: raise NoSuchElementException except AttributeError: raise NoSuchElementException else: first_page = False subforum_next_page = self.get_forum_subelement( ForumField.SUBFORUM_NEXT_PAGE, self.program.driver) if subforum_next_page is None: raise NoSuchElementException else: subforum_link = subforum_next_page.get_attribute( "href") if subforum_link is None: # Terrible, but should be worth it: if the current item has not the link, it will # likely be present in children. For this reason, I iterate on them until I don't find # the proper link. for child in subforum_next_page.find_elements_by_xpath( ".//*"): subforum_link = child.get_attribute("href") # Found it! if subforum_link != "" or not None: break # If I reach the end of for cycle, then there's a problem. raise NoSuchElementException self.program.driver.click(subforum_next_page) except NoSuchElementException: break else: # Scan next thread for thread, thread_post_count_by_browser in izip( self.get_forum_subelements(ForumField.THREAD_POOL, self.program.driver), self.get_forum_subelements( ForumField.THREAD_POST_COUNT, self.program.driver)): thread_post_count_by_browser_text = thread_post_count_by_browser.get_attribute( "text") if thread_post_count_by_browser_text is None: thread_post_count_by_browser_text = thread_post_count_by_browser.text thread_post_count_by_browser = filter( unicode.isdigit, thread_post_count_by_browser_text) thread_link = thread.get_attribute("href") if thread_link not in scanned_threads: scanned_threads.append(thread_link) do_i_have_to_scan_it = True has_ever_been_scanned = self.thread_list_dao.has_ever_been_scanned( thread_link) if has_ever_been_scanned: thread_post_count_by_db = str( self.thread_list_dao. get_post_count_by_thread_url(thread_link)) # The thread will count as scanned, whether will be carved or not, depending on post count. if thread_post_count_by_db == thread_post_count_by_browser: do_i_have_to_scan_it = False print_warning( "[CRAWLER] I already know thread " + thread_link + " and I've collected " "all the " + thread_post_count_by_browser + " posts in it. Skipped.") # The condition to satisfy is: has never been scanned or it has been scanned but different # post count. if do_i_have_to_scan_it: wait_simple_pause() self.program.driver.click(thread) # Begin thread crawling self.crawl_current_thread(thread_link, False) # Taking note of current amount of posts if has_ever_been_scanned: self.thread_list_dao.update_post_count( thread_link, thread_post_count_by_browser) else: self.thread_list_dao.insert_post_count( thread_link, thread_post_count_by_browser) # Done carving current thread, move on next going back to subforum self.program.driver.load_url(subforum_link) break # I need to exit the for loop since elements aren't anymore connected to the DOM
def crawl_current_thread(self, thread_link, is_second_try): # Thread crawl # This is needed for carving vbulletin threads, in order to tear apart first and last pages. print_warning("[CRAWLER] Starting to read posts on page...") try: # TODO use is_second_try for refreshing the page and continue the parsing from there first_page = True thread_name = self.get_element_text( self.get_thread_subelements(ThreadField.THREAD_TITLE, self.program.driver)[0]) while True: for thread_post in self.get_thread_subelements( ThreadField.POST_POOL, self.program.driver): # Post carving try: post_text = self.get_element_text( self.get_thread_subelement(ThreadField.POST_TEXT, thread_post)) date = self.get_element_text( self.get_thread_subelement(ThreadField.DATE, thread_post)) author = self.get_element_text( self.get_thread_subelement(ThreadField.AUTHOR, thread_post)) post_count = self.get_element_text( self.get_thread_subelement(ThreadField.POST_COUNT, thread_post)) if not self.post_dumps_dao.check_if_post_dump_exists( self.website_id, thread_link, thread_name, author, date, post_count, post_text): # TODO wait longer for the first post (50wpm) self.post_dumps_dao.insert_post_dump( self.website_id, thread_link, thread_name, author, date, post_count, post_text) wait_to_read(post_text) except StaleElementReferenceException as e: if not is_second_try: sleep(1) self.crawl_current_thread(thread_link, True) else: # This could be more worrying, but could also depend from a problem in the DOM of the page. print_error_and_exception( "[CRAWLER] Exception thrown. Might be a potential failure of " "the carving.", e) print_warning( "[CRAWLER] I'm reloading the page...") self.program.driver.load_url(thread_link) wait_simple_pause() # Try to reach next page of the thread, if any try: # vBulletin handles next pages differently compared to other forums. In particular, uses the same # tag and class to define the button of the next page, plus XPATH is not exploitable due the fact # it varies depending on thread length. if self.website_dao.get_is_vbullettin_by_website_id( self.website_id): prev_next_elements = self.get_thread_subelements( ThreadField.THREAD_NEXT_PAGE, self.program.driver) # If there is 4 navigation buttons (2 on top page, 2 on bottom page), or is the first page, # then there's a next page... if prev_next_elements.__len__() == 4 or ( prev_next_elements.__len__() == 2 and first_page): first_page = False self.program.driver.click(prev_next_elements[1]) # ...else, could be or a first or a last page. else: raise NoSuchElementException else: first_page = False thread_next_page = self.get_thread_subelement( ThreadField.THREAD_NEXT_PAGE, self.program.driver) if thread_next_page is None: raise NoSuchElementException else: try: self.program.driver.click(thread_next_page) #### TODO WORKAROUND PER OFFENSIVE COMMUNITY #### C'E' UNA ROBA NASCOSTA CHE CONTIENE LINK NON CORRETTI except ElementNotInteractableException: thread_next_page = self.get_thread_subelements( ThreadField.THREAD_NEXT_PAGE, self.program.driver)[1] self.program.driver.click(thread_next_page) except NoSuchElementException: break sleep(1) except (IndexError, TypeError) as e: if not is_second_try: sleep(1) self.crawl_current_thread(thread_link, True) else: print_error_and_exception( "[CRAWLER] Exception thrown. Might be a potential failure of the carving.", e)