def main():
# Read Files in Data Dir and get Dictionary
print "Reading Files... \n"
result_dict = parseFile.getDict('data/*')
# Get Json Array of Text
print "%d Records. Preparing Data... \n" % ( len(result_dict) )
textIdDict = buildJsonArray.getTextAndId(result_dict)
textJsonString = textIdDict["jsonTextList"]
idList = textIdDict["idList"]
textList = textIdDict["textList"]
authorList = textIdDict["authorList"]
# Sentiment Analysis on Text
print "Calling Sentiment Analysis API... \n"
sentimentJsonArray = apiCall.getSentimentResults(textJsonString)
# Associate Results with Id
print "Compiling Results... \n"
mySentimentDict = {}
for index in range( len(idList) ):
documentId = idList[index]
mySentimentDict[documentId] = sentimentJsonArray[index]
mySentimentDict[documentId]["commentText"] = textList[index]
mySentimentDict[documentId]["author"] = authorList[index]
# Sentiment Analysis on Text
print "Calling Sentiment Analysis API... \n"
sentimentJsonArray = apiCall.getSentimentResults(textJsonString)
# Associate Results with Id
print "Compiling Results... \n"
mySentimentDict = {}
for index in range( len(idList) ):
documentId = idList[index]
mySentimentDict[documentId] = sentimentJsonArray[index]
mySentimentDict[documentId]["commentText"] = textList[index]
mySentimentDict[documentId]["author"] = authorList[index]
# Write Sentiment Results to File
print "Writing to results/sentimentResults.json... \n"
fileIO.writeJsonFile(mySentimentDict, 'results/sentimentResults.json')
print "Done"
def main():
# Get API KEY as parameter to the function
if len(sys.argv) > 2:
API_KEY = sys.argv[1]
DOCKET_ID = sys.argv[2]
else:
print "SYNTAX: getData <API_KEY> <DOCKET_ID>"
quit()
# Get Number of Comments
commentData = apiCall.getLastNComments(API_KEY, DOCKET_ID, str(1000),
str(22000))
numberofComments = commentData["totalNumRecords"]
print "Fetching " + numberofComments + " comments."
# API limit is 1000. Call API 1+numberOfComments/1000 times
numberofIterations = int(numberofComments / 1000) + 1
for index in range(0, numberofIterations, 1):
PAGE_OFFSET = index * 1000
commentData = apiCall.getLastNComments(API_KEY, DOCKET_ID, str(1000),
str(PAGE_OFFSET))
fileIO.writeJsonFile(commentData, "data/file" + str(index))
print "Done."
def getGender(authorList):
counter = 0
apiLimit = 0
apiString = ""
myNameList = []
# Preprocess the data and use only
for index in range(len(authorList)):
counter += 1
fname = authorList[index]
myNameList.append(fname)
# Get only the first name
name = fname.split(' ')[0].lstrip(' ')
re.sub(r'\W+', '', name)
# Generate string in required API format
apiString = apiString + "name[%d]=%s&" % (counter, name)
# Check for API Limit
if apiLimit > 900:
break
# Set limit to 10 names per call
if (counter == 9):
# Call the API and write results to File
results = callGenderAPI(apiString)
myJsonDict = processResults(results, myNameList)
print "Processing \n"
# Reset counters after 10
apiString = ""
counter = 0
apiLimit += 10
myDocIdList = []
myNameList = []
# Call API last time for remaining names
if apiString != "":
callGenderAPI(apiString)
# Write Gender Results to File
print "Writing to results/genderResults.json... \n"
fileIO.writeJsonFile(myJsonDict, 'results/genderResults.json')
def getGender(authorList):
counter = 0
apiLimit = 0
apiString = ""
myNameList = []
# Preprocess the data and use only
for index in range ( len(authorList) ):
counter += 1
fname = authorList[index]
myNameList.append(fname)
# Get only the first name
name = fname.split(' ')[0].lstrip(' ')
re.sub(r'\W+', '', name)
# Generate string in required API format
apiString = apiString + "name[%d]=%s&" % (counter, name)
# Check for API Limit
if apiLimit > 900:
break
# Set limit to 10 names per call
if (counter == 9):
# Call the API and write results to File
results = callGenderAPI(apiString)
myJsonDict = processResults(results, myNameList)
print "Processing \n"
# Reset counters after 10
apiString = ""
counter = 0
apiLimit += 10
myDocIdList = []
myNameList = []
# Call API last time for remaining names
if apiString != "":
callGenderAPI(apiString)
# Write Gender Results to File
print "Writing to results/genderResults.json... \n"
fileIO.writeJsonFile(myJsonDict, 'results/genderResults.json')
