# create socket and connect to HTTP port of host

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

s.connect((host,80))

# send request

# request is a HTTPrequest object which is implemented by us, the string of the object gives the request to be sent

s.send(str(request))

# get response

# loop until blank string is returned

response = ""

while True:

received = s.recv(1000)

response = response + received

if received == "":

break

s.close() # close socket

# create a HTTPresponse object and parse the required response

httpresponse = HTTPresponse()

httpresponse.parse(response)

# parse link using urlparse

parsedLink = urlparse(login_link)

# create request using HTTPrequest object

request = HTTPrequest()

# set parameters of request

request.type = "GET"

request.host = parsedLink.netloc

request.path = parsedLink.path

request.version = "1.1"

request.connection = "close"

# send request and get response

httpresponse = sendRequest(parsedLink.netloc,request)

# parse login link

parsedLink = urlparse(login_link)

# set parameters for HTTPrequest object

loginrequest = HTTPrequest()

loginrequest.type = "POST"

loginrequest.version = "1.1"

loginrequest.host = parsedLink.netloc

loginrequest.path = parsedLink.path

loginrequest.connection = "Keep-Alive"

loginrequest.cookies['csrf'] = csrf # setting the csrf token from previous request

loginrequest.content_type = "application/x-www-form-urlencoded"

loginrequest.content = "username="+username+"&password="+password+"&csrfmiddlewaretoken="+csrf # the content

# send request and return response

loginresponse = sendRequest(parsedLink.netloc,loginrequest)

# to keep track of the links sent by the thread

sent_links = []

# the function to be executed by the sending thread of the crawler thread

# this simply sends requests for the links in the link_queue to implement pipelining

def sendRequests(csrf,session,s):

# repeat until link_queue is not empty

while link_queue.__len__() > 0:

# acquire mutex to prevent sync issues

mutex.acquire()

# pop link from queue

# can crash as queue could be empty due to other threads

try:

link = link_queue.pop(0)

except IndexError:

mutex.release() # release mutex and break

break

# if the popped link is not a visited link then send request to it

if link not in visited:

# make request object for it

parsed = urlparse(link)

linkrequest = HTTPrequest()

linkrequest.type = "GET"

linkrequest.version = "1.1"

linkrequest.host = parsed.netloc

linkrequest.path = parsed.path

linkrequest.cookies['csrf'] = csrf # send csrf token

linkrequest.cookies['sessionid'] = session # send session id

linkrequest.connection = "Keep-Alive" # for pipelining

# send the request

try:

s.send(str(linkrequest))

# if no error add link to sent links

sent_links.append(link)

except socket.error:

# if error occurs due to various reasons then insert link at start of queue

link_queue.insert(0,link)

mutex.release() # release mutex and break

#s.close()

break

mutex.release() # release mutex after successful transmission so that other threads can transmit

# function which processes responses

def process_response(response):

# if it is redirect then add the link to the visisted and insert the new link at start of queue

if 300 <= response.status < 400:

visited.append(sent_links.pop(0))

link_queue.insert(0,response.location)

# if it is internal error then insert the same link again at start of queue

elif 500 <= response.status < 600:

link = sent_links.pop(0)

link_queue.insert(0,link)

# if it is normal response then parse it and search for valid links and secret flags

elif response.status == 200 and response.status_message == 'OK':

# add to visited links

visited.append(sent_links.pop(0))

# parse the content of response

soup = BeautifulSoup(response.content,"html.parser")

# extract a and h2 tags with secret flag class

a_tags = soup.find_all('a')

h2_tags = soup.find_all('h2',{'class':'secret_flag'})

# add the flag in h2 tag if it is a new flag

for h2_tag in h2_tags:

if h2_tag.contents[0].split(" ")[1] not in secret_flags:

secret_flags.append(h2_tag.contents[0].split(" ")[1])

# loop through the a tags and add links if it is not in visited and a http link in fakebook

for a_tag in a_tags:

if urljoin(default_netloc,a_tag['href']) not in visited:

parsed = urlparse(a_tag['href'])

if (parsed.netloc == '' or parsed.netloc == urlparse(default_netloc).netloc) and \

(parsed.scheme == 'http' or parsed.scheme == ''):

if parsed.netloc == '':

link_queue.append(urljoin(default_netloc,a_tag['href']))

else:

link_queue.append(a_tag['href'])

# function to execute by receiving thread

def receive_responses(s):

# string which stores response

response = ""

# loop until blank string is received

while True:

try:

received = s.recv(1000) # receive response

except socket.error:

break # break if some error happens

response = response + received # append response

mutex.acquire() # acquire mutex

# break the response string into individual responses if 2 or more are present and process them

while response.count("HTTP/1.1") >= 2:

# response splitting code

httpresponse = HTTPresponse()

second_http_index = response.find("HTTP/1.1","HTTP/1.1".__len__())

one_response = response[0:second_http_index]

# process response

if not one_response == "":

httpresponse.parse(one_response)

process_response(httpresponse)

# update response to break from loop

response = response[second_http_index:]

mutex.release() # release mutex so that other threads can work

# break if blank string is received

if received == "":

break

# acquire mutex

mutex.acquire()

# do the same stuff like above

while response.count("HTTP/1.1") >= 2:

httpresponse = HTTPresponse()

second_http_index = response.find("HTTP/1.1","HTTP/1.1".__len__())

one_response = response[0:second_http_index]

if not one_response == "":

httpresponse.parse(one_response)

process_response(httpresponse)

response = response[second_http_index:]

# this is to process the final response left in the string response

if not response == "":

httpresponse = HTTPresponse()

httpresponse.parse(response)

process_response(httpresponse)

# incase a socket error happens there might be some links whose responses were not processed

# add these links back to the queue at the start so that they are not missed

if sent_links.__len__() > 0:

for sent_link in sent_links:

link_queue.insert(0,sent_link)

s.close() # close socket to signal the sending thread to stop and also since it is bad to leave a socket open

mutex.release() # release the mutex

# this is the logic which happens in crawler thread

# create a socket unique to the thread which connects to HTTP port of fakebook

thread_socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)

thread_socket.connect((host,80))

# this is for pipelining

# start sending thread to send requests

sending = Thread(target=sendRequests,args=(csrf,session_id,thread_socket))

sending.setDaemon(True)

sending.setName("Sending Thread for "+threading.current_thread.__name__)

sending.start()

# start receiving thread to receive and process responses

receiving = Thread(target=receive_responses, args=(thread_socket,))

receiving.setDaemon(True)

receiving.setName("Receiving Thread for "+threading.current_thread.__name__)