def crawl(cokie,sockObject,response): [stat,body,header_data]=webs.xtractInfo(response) urlCheck=webs.xtractLinks(body) i=0; for profileLink in urlCheck: url_unvisited.append(profileLink) if profileLink not in url_visited: profileLinkCrawl(profileLink,cokie,sockObject) url_visited.append(profileLink) i=i+1 try: url_unvisited.remove(profileLink) except: print("item not in list") else: continue while len(flag_list)<5: urlCheck=url_unvisited for profileLink in urlCheck: if profileLink in url_visited: continue else: profileLinkCrawl(profileLink,cokie,sockObject) url_visited.append(profileLink) i=i+1 try: url_unvisited.remove(profileLink) except: print("item not in list") urlCheck=url_unvisited sock2=socks.create_sock(host,port) sockObject=sock2 # print("The number of itiretions are "+str(i)) # print(url_unvisited) # print(url_visited) # print(flag_list) for flag in flag_list: print(flag)
def profileLinkCrawl(profileLink,cokie,sockObject): requestN=webs.reqGenCrawl(host,profileLink,userName,password,cokie) responseN=socks.handle_req(requestN,sockObject) [status,body,header_data]=webs.xtractInfo(responseN) while 1: if status=='500': sock1=socks.create_sock(host,port) requestN=webs.reqGenCrawl(host,profileLink,userName,password,cokie) responseN=socks.handle_req(requestN,sock1) [status2,body,header_data]=webs.xtractInfo(responseN) if status2=='200': sockObject=sock1 break else: status=status2 continue else: break try: urlList=webs.xtractLinks(body) if "<h2 class='secret_flag' style=" in body: flagCheck=webs.findSecretFlag(body) flag_list.append(flagCheck) for profileLink2 in urlList: if profileLink2 not in url_unvisited and profileLink not in url_visited: url_unvisited.append(profileLink2) except: sys.exit("Problem in profileLinkCrawl") return [status,body,header_data]