def test_remote_cache_check(self): arr = pickle.dumps(['http://yahoo.com','http://yahoo.com','3']) qUtils.send_message(arr, qUtils.CACHE_PORT) #self.myCache.main_cache() print "foo" response = qUtils.recv_message(qUtils.FETCH_PORT) print response time.sleep(3) qUtils.send_message('/localpath/tothe/file/yahoo', qUtils.FETCH_PORT) response2 = pickle.loads(qUtils.recv_message(qUtils.MGR_PORT)) print response2 response3 = qUtils.recv_message(qUtils.MGR_PORT) print response3
def run(self): print 'SECOND Thread' print 'QUERY 2 %s' %self.search_query search_results = self.execute_search(search_query) url_list_tmp = list(set(self.parse_results_for_urls(search_results))) url_list = [] for i in range (0,len(url_list_tmp)) : if i == 10 : break url_list.append(url_list_tmp[i]) print 'LIST %s'%url_list #using a lock here allows multiple requests to be serviced if url_list != [] : self.threadLock.acquire() qUtils.send_message(cPickle.dumps(url_list),qUtils.MGR_PORT) print 'SEND to MGR' data = cPickle.loads(qUtils.recv_message(qUtils.PROX_PORT)) print 'RECV %s'%data self.url_results = data self.threadLock.release()
def run(self): #ask to qCache if he has the data global web_list manager = qUtils.send_message(self.request,qUtils.CACHE_PORT) if self.request != "!KILLPROXY" : while True : data = qUtils.recv_message(qUtils.MGR_PORT) if data != 'END' : data = cPickle.loads(data) web_list.append(data) #add data to web_list else : break
class qManager(threading.Thread): def __init__(self,threadID, request): threading.Thread.__init__(self) self.threadID = threadID self.request = request def run(self): #ask to qCache if he has the data global web_list manager = qUtils.send_message(self.request,qUtils.CACHE_PORT) if self.request != "!KILLPROXY" : while True : data = qUtils.recv_message(qUtils.MGR_PORT) if data != 'END' : data = cPickle.loads(data) web_list.append(data) #add data to web_list else : break if __name__ == '__main__': while True: #receive proxy data url_list = qUtils.recv_message(qUtils.MGR_PORT) web_list = [] pThread = qManager(1,url_list) pThread.start() pThread.join() if url_list == "!KILLPROXY" : break print 'FIN THREAD %s'%web_list qUtils.send_message(cPickle.dumps(web_list),qUtils.PROX_PORT) print 'SEND TO PROXY'
self.imgs = list(set(self.soup.find_all('img'))) self.css = self.soup.find_all('link', rel="stylesheet") self.js = self.soup.find_all('script', src=re.compile(".*")) with open(self.saved_page_dir + '/index.html','w') as f: f.write(self.mainpage) #self.fetch_images() #self.fetch_css_and_js() #self.rewrite_links() print 'sending the cache a local file at ' + self.saved_page_dir + '/index.html' #qUtils.send_message(self.saved_page_dir+'/index.html', qUtils.CACHE_PORT) #wait for a fetch request from the cache, then dispatch a fetcher thread #and finally send the result back to the cache if __name__=='__main__': while True : #state = False #data = ' ' print 'LISTEN CACHE' data = qUtils.recv_message(qUtils.FETCH_PORT) if data == "!KILLPROXY": break pThread = qFetcherThread(5,data) pThread.start() time.sleep(1) pThread.join() qUtils.send_message(pThread.saved_page_dir + '/index.html', qUtils.CACHE_PORT)
print 'SEND to MGR' data = cPickle.loads(qUtils.recv_message(qUtils.PROX_PORT)) print 'RECV %s'%data self.url_results = data self.threadLock.release() if __name__ == '__main__': threadLock = threading.Lock() search_query = ' ' s = qUtils.open_listener(qUtils.BROWSER_PORT) print 'starting proxy ' while True: conn,addr = s.accept() request = conn.recv(8192) print request search_query = qProxyManager.parse_request_for_query(request) print 'QUERY %s'%search_query senddata = qUtils.build_standard_response([]) conn.sendall(senddata) if "KILLPROXY" in search_query: qUtils.send_message("!KILLPROXY", qUtils.MGR_PORT) break pThread = qProxyManager(1,search_query,threadLock) pThread.start() #pThread.join() if pThread.url_results != []: conn.sendall(qUtils.build_standard_response(pThread.url_results)) conn.close()
def run(self): print 'starting client thread' qUtils.send_message(self.message,self.port) return
def run(self): print "not in the cache " qUtils.send_message(self.request, qUtils.FETCH_PORT) if self.request == "!KILLPROXY" or self.request == 'END' : print 'END' else:
def run(self): qUtils.send_message(self.request, qUtils.MGR_PORT) # send the address to the mgr
pThread.start() pThread.join() print "send a file " if __name__ == '__main__': myTable = dict() while True: url_list = qUtils.recv_message(qUtils.CACHE_PORT) if url_list == "!KILLPROXY" : pThread = qCacheFetcher(2,url_list) pThread.start() pThread.join() break else: url_list = pickle.loads(url_list) print 'TABLE %s' %myTable for url in url_list : #take url if url in myTable: # Yes, send the local address to the manager print "yes in the cache" ayfile = pickle.dumps([url, myTable.get(url)]) pThread = qCacheManager(1,ayfile) pThread.start() pThread.join() else: pThread = qCacheFetcher(2,url) pThread.start() pThread.join() print 'FIN THREAD %s'%myTable qUtils.send_message("END", qUtils.MGR_PORT)