def _parse_contact_edges_v1(self, edges: list, hostuser: NetworkProfile, reason: str = None, get_profile_pic: bool = False) -> iter: """处理返回的edges字段""" try: for edge in edges: ctid = edge['node']['node']['id'] cturl = edge['node']['node']['url'] ctname = edge['node']['title']['text'] ct: NetworkProfile = NetworkProfile( ctname, ctid, self._SOURCE) ct.url = cturl ct.reason = reason if not reason is None else "facebook动态监测" if get_profile_pic: pic_url = edge['node']['image']['uri'] pic = self._ha.get_response_stream(pic_url) ct._profile_pic = helper_str.base64bytes(pic.read()) ct.set_contacts(hostuser, isfriend=True) hostuser.set_contacts(ct) yield ct except GeneratorExit: pass except: self._logger.error("Parse contacts edges failed: {}".format( traceback.format_exc()))
def get_banner_oracle( self, task: IscanTask, level, pinfo_dict, port, *args, zgrab2path: str = "zgrab2", sudo: bool = False, timeout: float = 600, ) -> iter: """scan oracle services and get the banner""" so: socket.socket = None if not isinstance(port, int) or port < 0 or port > 65535: raise Exception("Invalid port: {}".format(port)) # oracle 就直接用socket 去连一下,收完了就关了 for host, portinfo in pinfo_dict.items(): try: so = socket.socket(socket.AF_INET, socket.SOCK_STREAM) so.settimeout(10) # 30s timeout so.connect((host, port)) bss: bytes = bytes() while True: bs: bytes = so.recv(1024) if bs is None or len(bs) < 1: break bss += bs if len(bs) >= 1048576: # receive over size self._logger.warn( "Oracle banner grab received over 1MB data, terminating the connection: {}:{}" .format(portinfo._host, portinfo._port)) break banner = helper_str.base64bytes(bss) if not banner is None and banner != "": portinfo.set_oracle(banner) portinfo.banner = "Oracle:\n" + banner except: self._logger.error("Scan oracle error: {}:{} {}".format( host, port, traceback.format_exc())) continue finally: if not so is None and not so._closed: so.close()
def _base64_encrypt(cls, val, enc) -> str: # =?utf-8?b?xxxxx # 上面的代码已经判断完了,下面的代码就判断是否直接bytes加密或是str加密 res = None if isinstance(val, str): try: res = "=?{}?b?{}".format(enc, helper_str.base64str(val, enc)) except Exception: tmp = helper_str.repr_str(val) if not isinstance(val, str): raise Exception( "Invalid val for encryption: {}".format(val)) res = "=?{}?b?{}".format(enc, helper_str.base64str(tmp, enc)) elif isinstance(val, bytes): res = "=?{}?b?{}".format(enc, helper_str.base64bytes(val, enc)) else: raise Exception("Invalid val for encryption: {}".format(val)) return res
def set_favicon( self, data: [bytes, str], location_: str, hash_: str = None, ): """set favicon of the site\n data: the favicon image, could be bytes or base64-ed str\n location: the original url of the favicon image.\n hash: the hash code of the favicon image""" if data is None or not isinstance(location_, str) or location_ == "": raise Exception("Invalid favicon params") favstr = data if isinstance(data, bytes): favstr = helper_str.base64bytes(data, encoding='ascii') self.__favicon['data'] = favstr self.__favicon['location'] = location_ self.__favicon['hash'] = hash_
def _get_outputdict_group(self, jgroup: dict): """搞group""" jgroup["groupid"] = self._groupid jgroup["source"] = self._source if not self.groupname is None: jgroup["groupname"] = self.groupname if not self.grouptype is None: jgroup["grouptype"] = self.grouptype if not self.url is None: jgroup["url"] = self.url if not self.reason is None: jgroup["reason"] = self.reason if not self.membercount is None: jgroup["membercount"] = self.membercount if isinstance(self._profile_pic, bytes): jgroup["profile_pic"] = "=?utf-8?B?{}".format( helper_str.base64bytes(self._profile_pic)) if not self.details is None: jgroup["detail"] = self.details
def _get_outputdict_profile(self, jprofile: dict): """添加profile根节点""" jprofile["networkid"] = (self.nickname if self.nickname is not None else self._networkid) jprofile["userid"] = self._userid jprofile["source"] = self._source if self.url is not None: jprofile["url"] = self.url if self.reason is not None: jprofile["reason"] = self.reason if len(self._emails) > 0: jemail = jprofile["emails"] = [] with self._emails_locker: for e in self._emails: jemail.append({"email": e}) if len(self._phones) > 0: jphone = jprofile["phones"] = [] with self._phones_locker: for p in self._phones: jphone.append({"phone": p}) if self.nickname is not None: jprofile["nickname"] = self.nickname if self.gender is not None: jprofile["gender"] = self.gender if self.birthday is not None: jprofile["birthday"] = self.birthday if self.address is not None: jprofile["address"] = self.address if isinstance(self._profile_pic, bytes): jprofile["profile_pic"] = "=?utf-8?B?{}".format( helper_str.base64bytes(self._profile_pic)) elif isinstance(self._profile_pic, str) and self._profile_pic != "": jprofile["profile_pic"] = "=?utf-8?B?{}".format(self._profile_pic) if self._details is not None and len(self._details) > 0: jprofile["detail"] = json.dumps(self._details, ensure_ascii=False)
def _get_addrinfo(self, profile: NetworkProfile): """联系信息""" try: # education # https://www.facebook.com/profile.php?id=100030846743121&sk=about§ion=overview&lst=100013325533097%3A100030846743121%3A1568790537 url: str = "https://www.facebook.com/profile.php?id={}&sk=about§ion=contact-info&lst={}%3A{}%3A{}".format( profile._userid, self._userid, profile._userid, helper_time.ts_since_1970(10)) html = self._ha.getstring(url, headers=""" accept: */* accept-encoding: gzip, deflate accept-language: en-US,en;q=0.9 cache-control: no-cache content-type: application/x-www-form-urlencoded origin: https://www.facebook.com pragma: no-cache referer: {} sec-fetch-mode: cors sec-fetch-site: same-origin""".format(profile.url)) if html is None: return soup = BeautifulSoup( html.replace('<!--', '').replace('-->', ''), 'lxml') # photo photo = soup.select_one('._11kf.img') if photo: try: pic_url = photo.attrs['src'].replace('amp;', '') pic = self._ha.get_response_stream(pic_url) profile._profile_pic = helper_str.base64bytes(pic.read()) except: pass codes = soup.select('.hidden_elem code') for code in codes: str_code = str(code) code = BeautifulSoup(str_code, 'lxml') if str_code.__contains__('性别') or str_code.__contains__( '出生日期'): sex = re.findall(r'性别.*?class="_2iem">(.*?)</span>', str_code) if sex: profile.gender = sex[0] birth = re.findall(r'出生日期.*?class="_2iem">(.*?)</span>', str_code) if birth: profile.birthday = birth[0] elif str_code.__contains__('手机'): try: profile.set_phone( code.select_one('[class="_2iem"]').get_text( '--*--').split('--*--')[0].replace( '.', '').replace('-', '').replace(' ', '')) except: pass elif str_code.__contains__('出生日期'): profile.birthday = code.select_one( '[class="_2iem"]').get_text() except Exception: self._logger.error( "Get contact-info page failed: username:{} url:{}".format( profile._networkid, profile.url))
def _get_user_by_url(self, userurl: str, reason: str = None, get_profile_pic: bool = False, recursive: int = 0) -> NetworkProfile: """ensure the user by user url\n param recursive: 内部参数,不用管""" res: NetworkProfile = None try: html, redir = self._ha.getstring_unredirect(userurl, headers=""" accept: */* accept-encoding: gzip, deflate accept-language: en-US,en;q=0.9 cache-control: no-cache content-type: application/x-www-form-urlencoded origin: https://www.facebook.com pragma: no-cache referer: https://www.facebook.com/search/people/?q={}&epa=SERP_TAB sec-fetch-mode: cors sec-fetch-site: same-origin user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36""" ) if not redir is None and not redir == "": html = self._ha.getstring(redir, headers=""" accept: */* accept-encoding: gzip, deflate accept-language: en-US,en;q=0.9 cache-control: no-cache content-type: application/x-www-form-urlencoded origin: https://www.facebook.com pragma: no-cache referer: https://www.facebook.com/search/people/?q={}&epa=SERP_TAB sec-fetch-mode: cors sec-fetch-site: same-origin user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36""" ) userurl = redir if html is None: self._logger.error( "Access user homepage failed: {}".format(userurl)) return res if html.__contains__("You’re Temporarily Blocked"): # 被暂时屏蔽了,就递归等待,每次等10秒 # 等待2分钟(120秒)还没好就放弃,返回None # 后面需要搞账号池来解决此问题? if recursive >= 12: self._logger.info( "Search too fast, You’re Temporarily Blocked over 120s, give up." ) return None self._logger.info( "Search too fast, You’re Temporarily Blocked, sleep 10s") time.sleep(10) recursive += 1 res = self._get_user_by_url(userurl, reason=reason, get_profile_pic=get_profile_pic, recursive=recursive) return res # "entity_id":"100030846743121" # "profile_session_id":"100013325533097:100030846743121:1568721729" userid: str = None succ, userid = helper_str.substringif(html, 'entity_id":"', '"') if not succ: succ, userid = helper_str.substringif(html, 'profile_session_id":"', '"') if not succ: self._logger.error( "Match userid failed: {}".format(userurl)) return res else: userid = userid.split(':')[1] # type":"Person","name":"Jay Chou" username: str = None succ, username = helper_str.substringif(html, 'type":"Person","name":"', '"') if not succ: self._logger.error("Match username failed: {}".format(userurl)) return res res = NetworkProfile(username, userid, self._SOURCE) res.url = userurl if not reason is None: res.reason = reason else: res.reason = self._dtools.landing_facebook # profile photo if get_profile_pic: soup = BeautifulSoup( html.replace('<!--', '').replace('-->', ''), 'lxml') # photo photo = soup.select_one('._11kf.img') if photo: try: pic_url = photo.attrs['src'].replace('amp;', '') pic = self._ha.get_response_stream(pic_url) res._profile_pic = helper_str.base64bytes(pic.read()) except: pass except Exception: self._logger.error("Get user by url error: {}".format( traceback.format_exc())) return res
def _get_user_by_url_v1(self, userurl: str, reason: str = None, get_profile_pic: bool = False, recursive: int = 0) -> NetworkProfile: """ensure the user by user url\n param recursive: 内部参数,不用管""" res: NetworkProfile = None try: html, redir = self._ha.getstring_unredirect(userurl, headers=""" accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 accept-encoding: gzip, deflate accept-language: en-US,en;q=0.9 cache-control: no-cache content-type: application/x-www-form-urlencoded origin: https://www.facebook.com pragma: no-cache referer: https://www.facebook.com/search/people/?q={}&epa=SERP_TAB sec-fetch-mode: cors sec-fetch-site: same-origin user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36""" ) if not redir is None and not redir == "": html = self._ha.getstring(redir, headers=""" accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 accept-encoding: gzip, deflate accept-language: en-US,en;q=0.9 cache-control: no-cache content-type: application/x-www-form-urlencoded origin: https://www.facebook.com pragma: no-cache referer: https://www.facebook.com/search/people/?q={}&epa=SERP_TAB sec-fetch-mode: cors sec-fetch-site: same-origin user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36""" ) userurl = redir if html is None: self._logger.error( "Access user homepage failed: {}".format(userurl)) return res if html.__contains__("You’re Temporarily Blocked"): # 被暂时屏蔽了,就递归等待,每次等10秒 # 等待2分钟(120秒)还没好就放弃,返回None # 后面需要搞账号池来解决此问题? if recursive >= 12: self._logger.info( "Search too fast, You’re Temporarily Blocked over 120s, give up." ) return None self._logger.info( "Search too fast, You’re Temporarily Blocked, sleep 10s") time.sleep(10) recursive += 1 res = self._get_user_by_url_v1(userurl, reason=reason, get_profile_pic=get_profile_pic, recursive=recursive) return res # {"viewerID":"100054477585089","userVanity":"","userID":"100006113837332"} userid: str = None succ, userid = helper_str.substringif(html, '"userID":"', '"') if not succ: self._logger.error("Match userid failed: {}".format(userurl)) # <title>Kavishka D Wijesooriya</title> username: str = None succ, username = helper_str.substringif(html, '<title>', '</title>') if not succ: self._logger.error("Match username failed: {}".format(userurl)) return res res = NetworkProfile(username, userid, self._SOURCE) res.url = userurl if not reason is None: res.reason = reason else: res.reason = self._dtools.landing_facebook # profile photo # "profilePicNormal":{"uri":"*******"}, if get_profile_pic: succ, pic_url = helper_str.substringif( html, '"profilePicNormal":{"uri":"', '"}') if not succ: self._logger.error( "Get profile pic failed: {}".format(userurl)) return res try: pic_url = pic_url.replace('\\', '') pic = self._ha.get_response_stream(pic_url) res._profile_pic = helper_str.base64bytes(pic.read()) except: pass except Exception: self._logger.error("Get user by url error: {}".format( traceback.format_exc())) return res
def set_profile_pic(self, profile_pic: bytes): if not isinstance(profile_pic, bytes): raise Exception("Invalid profile_pic, should be bytes") self._profile_pic = helper_str.base64bytes(profile_pic)