def run(self): results = None if self.cf: print("[-] Still not implemented. Could not read from file.") else: try: url, params = self.build_request() hu = HTTPUtils(url, params=params) html = hu.make_request() links = ParsingEngine.parse_search(html.content, self.engine) if not links: print("[-] Search did not return results. Exiting...") exit() # Check if search with these params was made 4 this engine filepath = self.check_or_create_params_dir() filepath += "/" search_filepath = filepath # Check if last_result exist last_result = self.check_last_result(filepath) # Check if today, a search was done with these params filepath = self.check_or_create_today_dir(filepath) filepath += "/" i = 0 for link in links: try: if last_result == link: print("[ * Result from prior searches found * ]") break hu = HTTPUtils(link) html = hu.make_request() results = ParsingEngine.parse_result(html.content, self.engine) if not results: print("[-] No data retrieved for result: %s" % link) continue lines = self.build_text(results, link) f, e = self.check_or_create_res_dir(filepath, link, i) f += "/" if e: # result already saved continue self.download_photos(f, results) fu = FileUtils(f + "info.txt") if html.encoding: fu.write(lines.encode(html.encoding)) else: fu.write(lines) i += 1 except Exception as e: print("[-] Search result: %s could not be processed: %s" % (link, str(e)) ) import sys, os exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] print(exc_type, fname, exc_tb.tb_lineno) if links and links[0]: fu = FileUtils(search_filepath + "last_result") fu.write(links[0]) print("[+] %s results processed" % str(i)) except Exception as e: print("[-] Something went wrong: %s" % e) return results
def download_photos(self, filepath, results): if results.has_key("photos"): i = 0 for photo_link in results["photos"]: hu = HTTPUtils(photo_link) res = hu.make_request() photo_list = photo_link.split(".") ext = photo_list[len(photo_list) - 1] fu = FileUtils(filepath + str(i) + "." + ext) fu.write_binary(res.content) i += 1 else: print("[*] No photos to download.")