def gettweets(howmanytweets, dataoutputfile, logfile):
consumer_key = 'bqb2614rUsb9Ts7DhtyTaPuWI'
consumer_secret = 'Q9UMklSd41gMzUKIMzt5CzZQZNOS6E5ndHNGk58pUxBShvGO2r'
access_token = '925969087997542400-mqhVvH56uE8pA5sqprq6vCh7XVB00Wj'
access_secret = 'a9xyKdciNSoZdg6AapOA6774AVEYgnpId5eelD9DvXwRS'
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
api = tweepy.API(auth)
seqkeyfile = open('tweetsequencekey.txt')
tweetsequencekey = int(seqkeyfile.read())
seqkeyfile.close()
seqkeyfile = open('tweetsequencekey.txt', 'w')
tweetsequencenew = tweetsequencekey + howmanytweets + 1
seqkeyfile.write('%d\n' % (tweetsequencenew))
seqkeyfile.close()
count = 0
for result in limit_handled(
tweepy.Cursor(api.search, q=" mom ").items(), logfile):
jsonversion = json.dumps(result._json)
# print('\nJSONVERSION')
# print(jsonversion)
# print('\nJSONDUMPS')
# print(json.dumps(jsonversion, sort_keys=True, indent=4))
jsonstring = StringIO(jsonversion)
thisdict = json.load(jsonstring)
# print('\nTHISDICT')
# outstring = '%8d %s' % (count, thisdict)
# outputDAB.write(outstring + '\n')
outstring = 'KEY VALUE PAIRS COUNTER %5d' % (count)
printoutput(outstring, logfile)
for key, value in sorted(thisdict.items()):
outstring = '%8d XXZZXX %s XXZZXX %s ZZXXZZ' % (tweetsequencekey,
key, value)
dataoutputfile.write(outstring + '\n')
# outstring = 'ZZXXZZ'
# outputDAB.write(outstring + '\n')
# sys.exit()
# outputJSON.write(str(result))
# outputJSON.write("\n")
# outputText.write(result.text)
# outputText.write("\n")
# outputUserID.write(str(result.user.id))
# outputUserID.write("\n")
count += 1
tweetsequencekey += 1
# This is a kluge in here to limit execution so we can test code.
# In a real application this might go quite a while before exiting.
if count >= howmanytweets: break
def limit_handled(cursor, logfile):
while True:
try:
yield cursor.next()
except tweepy.RateLimitError:
outstring = "limit reached"
printoutput(outstring, logfile)
# time.sleep(15 * 60)
except tweepy.TweepError:
outstring = "limit reached"
printoutput(outstring, logfile)
def printsparsematrix(self, which):
""" This is the docstring.
"""
if which == 'TERM':
for key, value in sorted(self.sparsematrixbyterm.items()):
sss = ''
for item in value:
sss += ' {:5s}:{:6.4f}'.format(item[0], item[3])
printoutput('MATRIXTERM {:20s} {:s}'.format(key, sss), OUTFILE)
elif which == 'DOC':
for key, value in sorted(self.sparsematrixbydoc.items()):
sss = ''
for item in value:
sss += ' {:5s}:{:6.3f}'.format(item[0], item[3])
printoutput('MATRIXDOC {:6s} {:s}'.format(key, sss), OUTFILE)
def main(pathtodata):
""" This is the docstring.
"""
########################################################################
## Measure process and wall clock times.
dabtimer = DABTimer()
logstring = dabtimer.timecall('BEGINNING')
printoutput(logstring, LOGFILE)
theglobals = Globals()
theglobals.filenames = glob(pathtodata + '/*')
########################################################################
## Compute the term frequencies in each file.
computetermfreqs(theglobals)
########################################################################
## Filter from all the terms to only the most frequent.
for onedoc in theglobals.docs.values():
onedoc.filterformostfrequent(10)
########################################################################
## Compute the tfidf and the sparse matrices along the way.
computetfidf(theglobals)
########################################################################
## Print the results.
theglobals.printresults('SHORT')
########################################################################
## Now compute and print the cosine comparisons.
cosinedict = computecosinecomparison('DOCDOC',
theglobals.sparsematrixbydoc)
for key, value in sorted(cosinedict.items()):
sss = 'DOCDOC {:7s} '.format(key)
for item in sorted(value):
if item[1] >= 0.50:
sss += ' ({:6s} {:8.4f})'.format(item[0], item[1])
printoutput(sss, OUTFILE)
cosinedict = computecosinecomparison('TERMTERM',
theglobals.sparsematrixbyterm)
for key, value in sorted(cosinedict.items()):
sss = 'TERMTERM {:15s} '.format(key)
for item in sorted(value):
if item[1] >= 0.90:
sss += ' ({:6s} {:8.4f})'.format(item[0], item[1])
printoutput(sss, OUTFILE)
logstring = dabtimer.timecall('ENDING')
printoutput(logstring, LOGFILE)
def computecosinecomparison(label, sparsematrix):
""" This is the docstring.
"""
doclength = defaultdict(float)
for key, value in sparsematrix.items():
doclength[key] = computevectorlength(value)
for key, value in sorted(doclength.items()):
sss = '{:s} LENGTH {:20s} {:10.4f}'.format(label, key, value)
printoutput(sss, OUTFILE)
cosinedict = defaultdict(list)
for key1, value1 in sparsematrix.items():
for key2, value2 in sparsematrix.items():
# We don't want A versus B and also B versus A,
# and we don't want A versus A.
if key2 <= key1:
continue
product = 0.0
sss = '{:s} ONE {:s} {:s}'.format(label, key1, str(value1))
printoutput(sss, OUTFILE)
sss = '{:s} TWO {:s} {:s}'.format(label, key2, str(value2))
printoutput(sss, OUTFILE)
for item1 in value1:
# term1 = item1[0]
# tfidf1 = item1[1]
for item2 in value2:
# term2 = item2[0]
# tfidf2 = item2[1]
# if term1 == term2:
if item1[0] == item2[0]:
# addin = tfidf1 * tfidf2
addin = item1[1] * item2[1]
product += addin
product = product / doclength[key1]
product = product / doclength[key2]
printoutput(
'{:s} THR {:s} {:s} {:10.4f}\n'.format(label, key1, key2,
product), OUTFILE)
cosinedict[key1].append([key2, product])
cosinedict[key2].append([key1, product])
return cosinedict
def main(howmanytweets, dataoutputfilename, logfilename):
dataoutputfile = open(dataoutputfilename, 'w')
logfile = open(logfilename, 'w')
outstring = 'MAIN: GET %d TWEETS' % (howmanytweets)
printoutput(outstring, logfile)
outstring = "MAIN: WRITE TWEETS TO FILE '%s'" % (dataoutputfilename)
printoutput(outstring, logfile)
outstring = "MAIN: WRITE LOG TO FILE '%s'" % (logfilename)
printoutput(outstring, logfile)
gettweets(howmanytweets, dataoutputfile, logfile)
dataoutputfile.close()
def printresults(self, which):
""" This is the docstring.
"""
sss = 'LEGEND'
sss += '\nColumn 0: Filename'
sss += '\nColumn 1: Raw freq of the term in the doc'
sss += '\nColumn 2: Total word count for the doc'
sss += '\nColumn 3: (Raw freq)/(Total word count)'
sss += '\nColumn 4: Number of unique terms in the doc'
sss += '\nColumn 5: Number of unique terms in the collection'
sss += '\nColumn 6: Number of docs in which the term appears'
sss += '\nColumn 7: idf for this term'
sss += '\nColumn 8: tf-idf for this term'
sss += '\nColumn 9: The term'
printoutput(sss, OUTFILE)
for filename, onedoc in sorted(self.docs.items()):
justfilename = filename.split('/')[-1]
if which == 'ALL':
thetermlist = onedoc.allterms
elif which == 'SHORT':
thetermlist = onedoc.topterms
else:
sss = 'ERROR IN PRINTRESULTS {:s}'.format(which)
printoutput(sss, OUTFILE)
sys.exit()
for wordandpos, thisterm in sorted(thetermlist.items()):
thisterm = onedoc.allterms[wordandpos]
sss = '{:s} '.format(justfilename)
sss += '{:7d} '.format(thisterm.rawfreq)
sss += '{:7d} '.format(onedoc.totalwordcount)
sss += '{:10.4f} '.format(thisterm.tftf)
sss += '{:7d} '.format(len(onedoc.wordandposset))
sss += '{:7d} '.format(len(self.overallwordandposset))
sss += '{:7d} '.format(len(self.docset[wordandpos]))
sss += '{:10.4f} '.format(thisterm.idf)
sss += '{:10.4f} '.format(thisterm.tfidf)
sss += '{:s} '.format(wordandpos)
printoutput(sss, OUTFILE)
def dumptweet(label, thetweet, logfile):
outstring = '\nDUMPONETWEET %s' % (label)
printoutput(outstring, logfile)
for key, value in thetweet.items():
outstring = 'DUMPTWEET %s %8s %s' % (label, key, value)
printoutput(outstring, logfile)