def _parse_contact_edges_v1(self,
edges: list,
hostuser: NetworkProfile,
reason: str = None,
get_profile_pic: bool = False) -> iter:
"""处理返回的edges字段"""
try:
for edge in edges:
ctid = edge['node']['node']['id']
cturl = edge['node']['node']['url']
ctname = edge['node']['title']['text']
ct: NetworkProfile = NetworkProfile(
ctname, ctid, self._SOURCE)
ct.url = cturl
ct.reason = reason if not reason is None else "facebook动态监测"
if get_profile_pic:
pic_url = edge['node']['image']['uri']
pic = self._ha.get_response_stream(pic_url)
ct._profile_pic = helper_str.base64bytes(pic.read())
ct.set_contacts(hostuser, isfriend=True)
hostuser.set_contacts(ct)
yield ct
except GeneratorExit:
pass
except:
self._logger.error("Parse contacts edges failed: {}".format(
traceback.format_exc()))
def get_banner_oracle(
self,
task: IscanTask,
level,
pinfo_dict,
port,
*args,
zgrab2path: str = "zgrab2",
sudo: bool = False,
timeout: float = 600,
) -> iter:
"""scan oracle services and get the banner"""
so: socket.socket = None
if not isinstance(port, int) or port < 0 or port > 65535:
raise Exception("Invalid port: {}".format(port))
# oracle 就直接用socket 去连一下，收完了就关了
for host, portinfo in pinfo_dict.items():
try:
so = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
so.settimeout(10) # 30s timeout
so.connect((host, port))
bss: bytes = bytes()
while True:
bs: bytes = so.recv(1024)
if bs is None or len(bs) < 1:
break
bss += bs
if len(bs) >= 1048576: # receive over size
self._logger.warn(
"Oracle banner grab received over 1MB data, terminating the connection: {}:{}"
.format(portinfo._host, portinfo._port))
break
banner = helper_str.base64bytes(bss)
if not banner is None and banner != "":
portinfo.set_oracle(banner)
portinfo.banner = "Oracle:\n" + banner
except:
self._logger.error("Scan oracle error: {}:{} {}".format(
host, port, traceback.format_exc()))
continue
finally:
if not so is None and not so._closed:
so.close()
def _base64_encrypt(cls, val, enc) -> str:
# =?utf-8?b?xxxxx
# 上面的代码已经判断完了，下面的代码就判断是否直接bytes加密或是str加密
res = None
if isinstance(val, str):
try:
res = "=?{}?b?{}".format(enc, helper_str.base64str(val, enc))
except Exception:
tmp = helper_str.repr_str(val)
if not isinstance(val, str):
raise Exception(
"Invalid val for encryption: {}".format(val))
res = "=?{}?b?{}".format(enc, helper_str.base64str(tmp, enc))
elif isinstance(val, bytes):
res = "=?{}?b?{}".format(enc, helper_str.base64bytes(val, enc))
else:
raise Exception("Invalid val for encryption: {}".format(val))
return res
def set_favicon(
self,
data: [bytes, str],
location_: str,
hash_: str = None,
):
"""set favicon of the site\n
data: the favicon image, could be bytes or base64-ed str\n
location: the original url of the favicon image.\n
hash: the hash code of the favicon image"""
if data is None or not isinstance(location_, str) or location_ == "":
raise Exception("Invalid favicon params")
favstr = data
if isinstance(data, bytes):
favstr = helper_str.base64bytes(data, encoding='ascii')
self.__favicon['data'] = favstr
self.__favicon['location'] = location_
self.__favicon['hash'] = hash_
def _get_outputdict_group(self, jgroup: dict):
"""搞group"""
jgroup["groupid"] = self._groupid
jgroup["source"] = self._source
if not self.groupname is None:
jgroup["groupname"] = self.groupname
if not self.grouptype is None:
jgroup["grouptype"] = self.grouptype
if not self.url is None:
jgroup["url"] = self.url
if not self.reason is None:
jgroup["reason"] = self.reason
if not self.membercount is None:
jgroup["membercount"] = self.membercount
if isinstance(self._profile_pic, bytes):
jgroup["profile_pic"] = "=?utf-8?B?{}".format(
helper_str.base64bytes(self._profile_pic))
if not self.details is None:
jgroup["detail"] = self.details
def _get_outputdict_profile(self, jprofile: dict):
"""添加profile根节点"""
jprofile["networkid"] = (self.nickname if self.nickname is not None
else self._networkid)
jprofile["userid"] = self._userid
jprofile["source"] = self._source
if self.url is not None:
jprofile["url"] = self.url
if self.reason is not None:
jprofile["reason"] = self.reason
if len(self._emails) > 0:
jemail = jprofile["emails"] = []
with self._emails_locker:
for e in self._emails:
jemail.append({"email": e})
if len(self._phones) > 0:
jphone = jprofile["phones"] = []
with self._phones_locker:
for p in self._phones:
jphone.append({"phone": p})
if self.nickname is not None:
jprofile["nickname"] = self.nickname
if self.gender is not None:
jprofile["gender"] = self.gender
if self.birthday is not None:
jprofile["birthday"] = self.birthday
if self.address is not None:
jprofile["address"] = self.address
if isinstance(self._profile_pic, bytes):
jprofile["profile_pic"] = "=?utf-8?B?{}".format(
helper_str.base64bytes(self._profile_pic))
elif isinstance(self._profile_pic, str) and self._profile_pic != "":
jprofile["profile_pic"] = "=?utf-8?B?{}".format(self._profile_pic)
if self._details is not None and len(self._details) > 0:
jprofile["detail"] = json.dumps(self._details, ensure_ascii=False)
def _get_addrinfo(self, profile: NetworkProfile):
"""联系信息"""
try:
# education
# https://www.facebook.com/profile.php?id=100030846743121&sk=about§ion=overview&lst=100013325533097%3A100030846743121%3A1568790537
url: str = "https://www.facebook.com/profile.php?id={}&sk=about§ion=contact-info&lst={}%3A{}%3A{}".format(
profile._userid, self._userid, profile._userid,
helper_time.ts_since_1970(10))
html = self._ha.getstring(url,
headers="""
accept: */*
accept-encoding: gzip, deflate
accept-language: en-US,en;q=0.9
cache-control: no-cache
content-type: application/x-www-form-urlencoded
origin: https://www.facebook.com
pragma: no-cache
referer: {}
sec-fetch-mode: cors
sec-fetch-site: same-origin""".format(profile.url))
if html is None:
return
soup = BeautifulSoup(
html.replace('<!--', '').replace('-->', ''), 'lxml')
# photo
photo = soup.select_one('._11kf.img')
if photo:
try:
pic_url = photo.attrs['src'].replace('amp;', '')
pic = self._ha.get_response_stream(pic_url)
profile._profile_pic = helper_str.base64bytes(pic.read())
except:
pass
codes = soup.select('.hidden_elem code')
for code in codes:
str_code = str(code)
code = BeautifulSoup(str_code, 'lxml')
if str_code.__contains__('性别') or str_code.__contains__(
'出生日期'):
sex = re.findall(r'性别.*?class="_2iem">(.*?)</span>',
str_code)
if sex:
profile.gender = sex[0]
birth = re.findall(r'出生日期.*?class="_2iem">(.*?)</span>',
str_code)
if birth:
profile.birthday = birth[0]
elif str_code.__contains__('手机'):
try:
profile.set_phone(
code.select_one('[class="_2iem"]').get_text(
'--*--').split('--*--')[0].replace(
'.', '').replace('-', '').replace(' ', ''))
except:
pass
elif str_code.__contains__('出生日期'):
profile.birthday = code.select_one(
'[class="_2iem"]').get_text()
except Exception:
self._logger.error(
"Get contact-info page failed: username:{} url:{}".format(
profile._networkid, profile.url))
def _get_user_by_url(self,
userurl: str,
reason: str = None,
get_profile_pic: bool = False,
recursive: int = 0) -> NetworkProfile:
"""ensure the user by user url\n
param recursive: 内部参数，不用管"""
res: NetworkProfile = None
try:
html, redir = self._ha.getstring_unredirect(userurl,
headers="""
accept: */*
accept-encoding: gzip, deflate
accept-language: en-US,en;q=0.9
cache-control: no-cache
content-type: application/x-www-form-urlencoded
origin: https://www.facebook.com
pragma: no-cache
referer: https://www.facebook.com/search/people/?q={}&epa=SERP_TAB
sec-fetch-mode: cors
sec-fetch-site: same-origin
user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36"""
)
if not redir is None and not redir == "":
html = self._ha.getstring(redir,
headers="""
accept: */*
accept-encoding: gzip, deflate
accept-language: en-US,en;q=0.9
cache-control: no-cache
content-type: application/x-www-form-urlencoded
origin: https://www.facebook.com
pragma: no-cache
referer: https://www.facebook.com/search/people/?q={}&epa=SERP_TAB
sec-fetch-mode: cors
sec-fetch-site: same-origin
user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36"""
)
userurl = redir
if html is None:
self._logger.error(
"Access user homepage failed: {}".format(userurl))
return res
if html.__contains__("You’re Temporarily Blocked"):
# 被暂时屏蔽了，就递归等待，每次等10秒
# 等待2分钟（120秒）还没好就放弃，返回None
# 后面需要搞账号池来解决此问题？
if recursive >= 12:
self._logger.info(
"Search too fast, You’re Temporarily Blocked over 120s, give up."
)
return None
self._logger.info(
"Search too fast, You’re Temporarily Blocked, sleep 10s")
time.sleep(10)
recursive += 1
res = self._get_user_by_url(userurl,
reason=reason,
get_profile_pic=get_profile_pic,
recursive=recursive)
return res
# "entity_id":"100030846743121"
# "profile_session_id":"100013325533097:100030846743121:1568721729"
userid: str = None
succ, userid = helper_str.substringif(html, 'entity_id":"', '"')
if not succ:
succ, userid = helper_str.substringif(html,
'profile_session_id":"',
'"')
if not succ:
self._logger.error(
"Match userid failed: {}".format(userurl))
return res
else:
userid = userid.split(':')[1]
# type":"Person","name":"Jay Chou"
username: str = None
succ, username = helper_str.substringif(html,
'type":"Person","name":"',
'"')
if not succ:
self._logger.error("Match username failed: {}".format(userurl))
return res
res = NetworkProfile(username, userid, self._SOURCE)
res.url = userurl
if not reason is None:
res.reason = reason
else:
res.reason = self._dtools.landing_facebook
# profile photo
if get_profile_pic:
soup = BeautifulSoup(
html.replace('<!--', '').replace('-->', ''), 'lxml')
# photo
photo = soup.select_one('._11kf.img')
if photo:
try:
pic_url = photo.attrs['src'].replace('amp;', '')
pic = self._ha.get_response_stream(pic_url)
res._profile_pic = helper_str.base64bytes(pic.read())
except:
pass
except Exception:
self._logger.error("Get user by url error: {}".format(
traceback.format_exc()))
return res
def _get_user_by_url_v1(self,
userurl: str,
reason: str = None,
get_profile_pic: bool = False,
recursive: int = 0) -> NetworkProfile:
"""ensure the user by user url\n
param recursive: 内部参数，不用管"""
res: NetworkProfile = None
try:
html, redir = self._ha.getstring_unredirect(userurl,
headers="""
accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
accept-encoding: gzip, deflate
accept-language: en-US,en;q=0.9
cache-control: no-cache
content-type: application/x-www-form-urlencoded
origin: https://www.facebook.com
pragma: no-cache
referer: https://www.facebook.com/search/people/?q={}&epa=SERP_TAB
sec-fetch-mode: cors
sec-fetch-site: same-origin
user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36"""
)
if not redir is None and not redir == "":
html = self._ha.getstring(redir,
headers="""
accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
accept-encoding: gzip, deflate
accept-language: en-US,en;q=0.9
cache-control: no-cache
content-type: application/x-www-form-urlencoded
origin: https://www.facebook.com
pragma: no-cache
referer: https://www.facebook.com/search/people/?q={}&epa=SERP_TAB
sec-fetch-mode: cors
sec-fetch-site: same-origin
user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36"""
)
userurl = redir
if html is None:
self._logger.error(
"Access user homepage failed: {}".format(userurl))
return res
if html.__contains__("You’re Temporarily Blocked"):
# 被暂时屏蔽了，就递归等待，每次等10秒
# 等待2分钟（120秒）还没好就放弃，返回None
# 后面需要搞账号池来解决此问题？
if recursive >= 12:
self._logger.info(
"Search too fast, You’re Temporarily Blocked over 120s, give up."
)
return None
self._logger.info(
"Search too fast, You’re Temporarily Blocked, sleep 10s")
time.sleep(10)
recursive += 1
res = self._get_user_by_url_v1(userurl,
reason=reason,
get_profile_pic=get_profile_pic,
recursive=recursive)
return res
# {"viewerID":"100054477585089","userVanity":"","userID":"100006113837332"}
userid: str = None
succ, userid = helper_str.substringif(html, '"userID":"', '"')
if not succ:
self._logger.error("Match userid failed: {}".format(userurl))
# <title>Kavishka D Wijesooriya</title>
username: str = None
succ, username = helper_str.substringif(html, '<title>',
'</title>')
if not succ:
self._logger.error("Match username failed: {}".format(userurl))
return res
res = NetworkProfile(username, userid, self._SOURCE)
res.url = userurl
if not reason is None:
res.reason = reason
else:
res.reason = self._dtools.landing_facebook
# profile photo
# "profilePicNormal":{"uri":"*******"},
if get_profile_pic:
succ, pic_url = helper_str.substringif(
html, '"profilePicNormal":{"uri":"', '"}')
if not succ:
self._logger.error(
"Get profile pic failed: {}".format(userurl))
return res
try:
pic_url = pic_url.replace('\\', '')
pic = self._ha.get_response_stream(pic_url)
res._profile_pic = helper_str.base64bytes(pic.read())
except:
pass
except Exception:
self._logger.error("Get user by url error: {}".format(
traceback.format_exc()))
return res
def set_profile_pic(self, profile_pic: bytes):
if not isinstance(profile_pic, bytes):
raise Exception("Invalid profile_pic, should be bytes")
self._profile_pic = helper_str.base64bytes(profile_pic)
