예제 #1
0
def initiate():
	global count,  search_max_depth
	while(URLProvider.isIncomplete()):
		curr_url = URLProvider.get_URL()
		if(int(curr_url[1]) <= int(search_max_depth) ):
			#Print can be removed
			print curr_url, count
			my_tuple=('00000',{'customError':'NO ACCESS'},None,None,0,0)
			if(SiteHandler.is_Valid(curr_url[0])):
					#Process further
					response_data=URLFetcher.getURL(curr_url[0])
					if(response_data==1):
						my_tuple=('2222',{'customError':'Unable to Fetch Correctly'}, None, None,0, curr_url[1])
					else:
						myMime =response_data.headers.get('Content-Type')
						if(valid_mime_type(myMime)):
							base_url=utils.getBaseUrl(curr_url[0])
							mystring = response_data.read()
							links = AttributeExtraction.extract_Links(base_url,mystring)
							stored_tup = pagestoragehandler.write_log(curr_url[0],mystring)
							my_tuple = (response_data.code, response_data.headers, stored_tup, len(links), len(mystring),curr_url[1])
							#links or None
							for link in set(links):
								if(valid_protocol_request(urlsplit(link)[0])):
									if(infologger.not_in_list(link)):
										URLProvider.add_URL(link)
						else:
							#We need to keep a log of Data-Stored. If a log of data downloaded is required
							#We can put the response object's data to find its size. 
							#Ideally headers do provide the size in headers, but we found some cases
							#it was not provided. So implemented it in this fashion.
							my_tuple=('1111',{'customError':'Unsupported Mime'}, None, None,0, curr_url[1])
						
			infologger.write_summary(curr_url[0], my_tuple )
			count += 1
						
	# Force writes and commits
	pagestoragehandler.store_log()
	infologger.store_log()
예제 #2
0
def initial_seeding(url_list):
	count = 0
	for url in url_list:
		URLProvider.add_URL(url_list[count])
		count += 1