def scanfolder():

	biglist = []

	for category in os.listdir('./dict'):
		print("Category: " + category)
		documents = os.listdir('./dict/' + category)[:900]
		
		stripped_documents = [Utils.testStripping('./dict/' + category + '/' + doc) for doc in documents]
		
		for strip_doc in stripped_documents:
			biglist.extend(strip_doc)
		
		
	# Remove duplicates
	biglist = list(set(biglist))
	bigdict = {}
	
	for word in biglist:
		bigdict[word] = 0
	
	
	with open( "./mainDict.json", "w" ) as outfile:
		json.dump(bigdict, outfile, indent=4, separators=(',', ': '))
	
		


	return
	os.chdir("dict")
	
	for name in glob.glob('*/*'):
		bigdict.extend(Utils.testStripping(name))

	#print(bigdict)
	print(' Current length  : ',len(bigdict))
	print(' Removing dupes : ')
	finalList = list(set(bigdict))
	print(' New size : ', len(finalList))


	# Make dict
	d = {}
	# Save to disk as dict
	for item in finalList:
		d[item] = 0

	with open( "..\\mainDict.json", "w" ) as outfile:
		json.dump(d, outfile)


	print ("Total length : ",len(d))
def sp_getTextDocument(path):

	words = Utils.testStripping(path)
	result = {}
	for word in words:

		try:
			result[word]
			result[word] = result['word'] + 1
		except:
			result[word] = 1

	return result
def getExampleArticlesFromSubCat(subcat):
	# Ex: subcat  = dict/rec.autos

	# Pick n-learning articles from subset.
	lst = os.listdir(subcat)
	lst = lst[:numberOfLearningArticlesInSubset]

	# Open each file and add it to a list of wooooords.
	subsetWords = []

	for fname in lst:
		subsetWords.extend(Utils.testStripping(subcat + "/" + fname))

	# Many words we got, time to do some counting.
	print("Subset word count: ", len(subsetWords))
	return subsetWords
def getTextDocument(path):
    return Utils.testStripping(path)