def crawl(cokie,sockObject,response):
[stat,body,header_data]=webs.xtractInfo(response)
urlCheck=webs.xtractLinks(body)
i=0;
for profileLink in urlCheck:
url_unvisited.append(profileLink)
if profileLink not in url_visited:
profileLinkCrawl(profileLink,cokie,sockObject)
url_visited.append(profileLink)
i=i+1
try:
url_unvisited.remove(profileLink)
except:
print("item not in list")
else:
continue
while len(flag_list)<5:
urlCheck=url_unvisited
for profileLink in urlCheck:
if profileLink in url_visited:
continue
else:
profileLinkCrawl(profileLink,cokie,sockObject)
url_visited.append(profileLink)
i=i+1
try:
url_unvisited.remove(profileLink)
except:
print("item not in list")
urlCheck=url_unvisited
sock2=socks.create_sock(host,port)
sockObject=sock2
# print("The number of itiretions are "+str(i))
# print(url_unvisited)
# print(url_visited)
# print(flag_list)
for flag in flag_list:
print(flag)
def profileLinkCrawl(profileLink,cokie,sockObject):
requestN=webs.reqGenCrawl(host,profileLink,userName,password,cokie)
responseN=socks.handle_req(requestN,sockObject)
[status,body,header_data]=webs.xtractInfo(responseN)
while 1:
if status=='500':
sock1=socks.create_sock(host,port)
requestN=webs.reqGenCrawl(host,profileLink,userName,password,cokie)
responseN=socks.handle_req(requestN,sock1)
[status2,body,header_data]=webs.xtractInfo(responseN)
if status2=='200':
sockObject=sock1
break
else:
status=status2
continue
else:
break
try:
urlList=webs.xtractLinks(body)
if "<h2 class='secret_flag' style=" in body:
flagCheck=webs.findSecretFlag(body)
flag_list.append(flagCheck)
for profileLink2 in urlList:
if profileLink2 not in url_unvisited and profileLink not in url_visited:
url_unvisited.append(profileLink2)
except:
sys.exit("Problem in profileLinkCrawl")
return [status,body,header_data]
