def sendRequest(host,request): # create socket and connect to HTTP port of host s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((host,80)) # send request # request is a HTTPrequest object which is implemented by us, the string of the object gives the request to be sent s.send(str(request)) # get response # loop until blank string is returned response = "" while True: received = s.recv(1000) response = response + received if received == "": break s.close() # close socket # create a HTTPresponse object and parse the required response httpresponse = HTTPresponse() httpresponse.parse(response) return httpresponse
def receive_responses(s): # string which stores response response = "" # loop until blank string is received while True: try: received = s.recv(1000) # receive response except socket.error: break # break if some error happens response = response + received # append response mutex.acquire() # acquire mutex # break the response string into individual responses if 2 or more are present and process them while response.count("HTTP/1.1") >= 2: # response splitting code httpresponse = HTTPresponse() second_http_index = response.find("HTTP/1.1","HTTP/1.1".__len__()) one_response = response[0:second_http_index] # process response if not one_response == "": httpresponse.parse(one_response) process_response(httpresponse) # update response to break from loop response = response[second_http_index:] mutex.release() # release mutex so that other threads can work # break if blank string is received if received == "": break # acquire mutex mutex.acquire() # do the same stuff like above while response.count("HTTP/1.1") >= 2: httpresponse = HTTPresponse() second_http_index = response.find("HTTP/1.1","HTTP/1.1".__len__()) one_response = response[0:second_http_index] if not one_response == "": httpresponse.parse(one_response) process_response(httpresponse) response = response[second_http_index:] # this is to process the final response left in the string response if not response == "": httpresponse = HTTPresponse() httpresponse.parse(response) process_response(httpresponse) # incase a socket error happens there might be some links whose responses were not processed # add these links back to the queue at the start so that they are not missed if sent_links.__len__() > 0: for sent_link in sent_links: link_queue.insert(0,sent_link) s.close() # close socket to signal the sending thread to stop and also since it is bad to leave a socket open mutex.release() # release the mutex
def sendRequest(host,request): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((host,80)) s.send(str(request)) response = "" while True: received = s.recv(1000) response = response + received if received == "": break s.close() httpresponse = HTTPresponse() httpresponse.parse(response) return httpresponse