def post(self): # you should get some argument from follow data = self.get_argument("data") if_zip = self.get_argument("if_zip") if if_zip == 'true': if_zip = True mac = self.get_argument("mac") msgs = unpackage({ 'data': data, 'if_zip': if_zip, 'mac': mac, }) db = Cache(os.path.expanduser(MSG_DB)) db.save_all(*msgs) # ..... # for parse json post # post_args = json.loads(self.request.body.decode("utf8", "ignore"))['msg'] # redirect or reply some content # self.redirect() self.write(json.dumps({'ok': len(msgs)})) self.finish()
class Fofa: def __init__(self, proxy=False): self.main_url = "https://fofa.so" self.flow = FLowNet("https://fofa.so", proxy=proxy) self.waiter = WebDriverWait(self.flow.phantom, 12) self.LOGIN_INFO = LOGIN_INFO self.LOGIN_INFO_SESSION = LOGIN_INFO_SESSION self.DB_Handle = Cache(DB_FOFA) def selector(self, x, time=0): try: r = self.waiter.until( EC.presence_of_element_located((By.CSS_SELECTOR, x))) if r: return r except TimeoutException as e: if isinstance(time, int) and time > 3: rprint("not found : %s , you can see error page in " % x + colored("/tmp/TimeoutException.png ", 'red')) self.flow.screenshot("TimeoutException") with open("/tmp/TimeoutException.html", "w") as fp: fp.write(self.flow.html()) # raise TimeoutException("time out not found : %s" %x) sys.exit(0) return self.selector(x, time + 1) def load_session_login(self): if os.path.exists(self.LOGIN_INFO_SESSION): self.load_session() else: self.login() def login(self, u=None): user = None passwd = None if not u: if os.path.exists(self.LOGIN_INFO): with open(LOGIN_INFO) as fp: u = json.load(fp) user = u['username'] passwd = u['password'] else: user = input("email>") passwd = getpass.getpass("passwd>") else: user = u['username'] passwd = u['password'] gprint("try login") #selector = lambda x: waiter.until(EC.presence_of_element_located((By.CSS_SELECTOR, x))) to_login = self.selector("a#but_zc") self.flow.screenshot('login') to_login.click() input_user = self.selector('#username') input_pass = self.selector('#password') input_user.send_keys(user) input_pass.send_keys(passwd + "\n") gprint(" --- Login ---") # with open(LOGIN_INFO_SESSION, 'w') as fp: u = {'username': user, 'password': passwd} with open(self.LOGIN_INFO, 'w') as fp: json.dump(u, fp) self.save_session() return user, passwd def save_session(self): gprint(" save session cookies") self.flow.go(self.main_url) with open(self.LOGIN_INFO_SESSION, 'wb') as fp: pickle.dump(self.flow.phantom.get_cookies(), fp) def load_session(self): gprint(" load cookies from") cookies = pickle.load(open(self.LOGIN_INFO_SESSION, "rb")) self.flow.go(self.main_url) self.flow.screenshot("beforelogin") for cookie in cookies: gprint(str(cookie)) try: self.flow.phantom.add_cookie(cookie) except Exception as e: rprint("load cookie failed. try --login") def search(self, key, page=0): search_input = self.selector("input#q") search_input.send_keys("{}\n".format(key)) self.flow.screenshot("mod") check_if_found = self.selector(".list_jg") check_if_found = check_if_found.get_attribute('outerHTML') num = 0 try: num = re.findall(r'Total\ results:\s(\d+)', check_if_found)[0] gprint("found result in :%s = %s" % (key, num)) num = int(num) if num == 0: return except IndexError: self.flow.screenshot("error.png") rprint("page load error !! see /tmp/error.png ") return self.selector(".list_mod") res = self.flow.html() iis = [] iis += self.parse(res) gprint("Done %d" % len(iis)) if len(iis) >= num: return iis gprint(" page 1") if page and isinstance(page, int): for i in range(page): next_page = self.selector("a.next_page") next_page.click() self.selector(".list_mod") res = self.flow.html() gprint(" page " + str(i + 2)) iis += self.parse(res) return iis def parse(self, res): mods = BS(res, 'html.parser').select('.list_mod') infos = [] c = 0 for m in mods: c += 1 ports = '/'.join([ i.text for i in m.select('.list_mod_t > .span > span') ]).replace("\n", "").replace(" ", "") pa = m.select(".list_sx1 > li") rip = m.select("li > i.fa-map-marker") rtime = m.select("li > i.fa-clock-o") rplane = m.select("li > i.fa-plane") rhost = m.select(".list_mod_t > a") ros = m.select("li > span.list_xs2") ip = "" ti = "" time = "" geo = "" os = "" host = "" if rip: ip = rip[0].parent.text.replace("\n", "").replace(" ", "") if rtime[0].parent.text != pa[0].text: ti = pa[0].text.replace('\n', '').replace(' ', '') if rtime: time = rtime[0].parent.text.replace('\n', '').replace(' ', '') if rplane: geo = rplane[0].parent.text.replace('\n', '').replace(' ', '') if rhost: host = rhost[0].attrs['href'] if ros: os = ros[0].parent.text.replace("\n", "").replace(" ", "") body = m.select('.auto-wrap')[0] if len(list(body)) > 1: body = ''.join([i.__str__() for i in list(body)]) else: body = body.text lprint(title=ti, host=host, ip=ip, ports=ports, os=os, time=time, geo=geo, body=body) info = Info(title=ti, host=host, ip=ip, ports=ports, os=os, ctime=time, geo=geo, body=body) infos.append(info) iis = [] for i in infos: # gprint(i.ip) if not self.DB_Handle.query_one( Info, m='and', ip=ip, ports=ports, ctime=time): iis.append(i) gprint("-- save %d --" % len(iis)) self.DB_Handle.save_all(*iis) return infos