def getPrefixesOfAlexaSuffixList(PSList, dumpFile):
	prefixCounter = {}
	for line in dumpFile:
		match = re.match(r'\d+,(.*?)$', line)
		if match:
			domain = match.group(1)
		else:
			continue
		if processDump.isIp(domain):
			continue
		buildPrefixCounter(domain, PSList, prefixCounter)
	return prefixCounter
def getPrefixesOfSuffixList(PSList, dumpFile):
  # map  suffix => prefixes of suffix
	prefixCounter = {}
	for line in dumpFile:
		record = processDump.getRecord(line)
		url, referrer, method, status, \
			request_cookie, response_cookie = record
    
		domain = getDomain(url)
    # dont process IPs
		if processDump.isIp(domain):
			continue
		buildPrefixCounter(domain, PSList, prefixCounter)
	return prefixCounter
def getInitialPSList(dumpFile):	
	PSList = set()
	currentLength = 1
	for line in dumpFile:	
		record = processDump.getRecord(line)
		url, referrer, method, status, \
			request_cookie, response_cookie = record

		urlDomain, urlMime, urlQuery, urlParams = \
		    processDump.parseURL(url)
	  # dont process IP addresses	
		if processDump.isIp(urlDomain):
			continue
		urlDomain = suffixParser.get_public_suffix(urlDomain)
		if len(urlDomain.split('.')) > currentLength:
			PSList.add(urlDomain)
	return PSList
def getInitialPSListFromAlexa(dumpFile):
	PSList = set()
	currentLength = 1
	suffixParser = publicsuffix.PublicSuffixList()
	for line in dumpFile:	
		match = re.match(r'\d+,(.*?)$', line)
		if match:
			urlDomain = match.group(1)
		else:
			continue
		if processDump.isIp(urlDomain):
			continue
	
		urlDomain = suffixParser.get_public_suffix(urlDomain)
		if len(urlDomain.split('.')) > currentLength:
			PSList.add(urlDomain)
	return PSList
def getNextLevelAlexa(dumpFile, PSList):
  candidatePSList = set()
	for line in dumpFile:	
		match = re.match(r'\d+,(.*?)$', line)
		if match:
			urlDomain = match.group(1)
		else:
			continue
		if processDump.isIp(urlDomain):
			continue

		currentDotIndex = urlDomain.find('.') 
		previousDotIndex = -1
		while currentDotIndex != -1:
			suffix = urlDomain[currentDotIndex + 1:]
			if suffix in PSList:
				prefix = urlDomain[previousDotIndex + 1:currentDotIndex]	
				candidatePSList.add('.'.join([prefix, suffix]))
			previousDotIndex = currentDotIndex
			currentDotIndex = urlDomain.find('.', currentDotIndex + 1)
def getNextLevel(dumpFile, PSList):
  candidatePSList = set()
  for line in dumpFile:
		record = processDump.getRecord(line)
		url, referrer, method, status, \
			request_cookie, response_cookie = record

		urlDomain, urlMime, urlQuery, urlParams = \
		    processDump.parseURL(url)
	  # dont process IP addresses	
		if processDump.isIp(urlDomain):
			continue

		currentDotIndex = urlDomain.find('.') 
		previousDotIndex = -1
		while currentDotIndex != -1:
			suffix = urlDomain[currentDotIndex + 1:]
			if suffix in PSList:
				prefix = urlDomain[previousDotIndex + 1:currentDotIndex]	
				candidatePSList.add('.'.join([prefix, suffix]))
			previousDotIndex = currentDotIndex
			currentDotIndex = urlDomain.find('.', currentDotIndex + 1)
  return candidatePSList