Exemplo n.º 1
0
def timelapse(url,
              dateRange=None,
              screen_name='',
              tweetID='',
              musicTrack='',
              startTime=-1):

    someThingWentWrongFlag = False
    if (len(url) > 0):
        url = url.lower()
        mementoGIFsPath, folderAlreadyExists = generateFolderNameFromURLUsingHash(
            url)

        #if folder exists exit - start

        if (folderAlreadyExists):
            print('... folder already exists, exiting')
            return False

        #if folder exists exit - end

        if (len(mementoGIFsPath) > 0):

            try:

                possibleFolderNameToDelete = os.getcwd(
                ) + '/' + mementoGIFsPath + '/'

                print("...opening urlsFile.txt")
                urlsFile = open("./" + mementoGIFsPath + "/urlsFile.txt", "w")

                # scrutiny - start
                print("...getting memento pages")
                pages = getMementosPages(url)
                print("...done getting memento pages")

                if len(pages) > 0:
                    mementosList = []
                    for i in range(0, len(pages)):
                        mementos = getItemGivenSignature(pages[i])

                        # filter mementos by date range if provided
                        if dateRange:
                            mementos = filterMementosWithDateRange(
                                mementos, dateRange)

                        mementosList.append(mementos)
                    # scrutiny - end

                    mementosList = sorted(mementosList, key=len)

                    numOfURLPosts = getNumOfURLPosts(url)
                    yearUrlDictionary = {}

                    yearUrlDictionary = get1MementoPerYear(
                        yearUrlDictionary, mementosList,
                        globalMementoUrlDateTimeDelimeter, numOfURLPosts)

                    # get mementos per month if the URL has mementos for only one year
                    if len(yearUrlDictionary) < 2:
                        yearUrlDictionary = get1MementoPerMonth(
                            {}, mementosList,
                            globalMementoUrlDateTimeDelimeter, numOfURLPosts)

                    #sort by date
                    #this does not seem to return dictionary, list instead
                    sortedKeysOfYearUrlDictionary = sorted(
                        yearUrlDictionary.keys())

                    lengthOfYearUrlDictionary = len(yearUrlDictionary)
                    if (lengthOfYearUrlDictionary > 0):

                        beginYear = str(sortedKeysOfYearUrlDictionary[0])
                        endYear = str(sortedKeysOfYearUrlDictionary[
                            len(sortedKeysOfYearUrlDictionary) - 1])

                        if dateRange:
                            dates = dateRange.split(' - ')
                            beginYear = dates[0]
                            endYear = dates[1]

                        urlsFile.write("What Did " + url + " Look Like From " +
                                       beginYear + " To " + endYear + "?\n\n")
                        urlsFile.write("Links" + ":\n")

                    #print len(yearUrlDictionary), " years "

                    #for year,url in yearUrlDictionary.items():
                    #	print year, ",", url

                    print("...taking screenshots")
                    result = takeScreenshots(yearUrlDictionary,
                                             mementoGIFsPath, urlsFile)
                    print("...done taking screenshots")

                    print("...done opening urlsFile.txt")
                    urlsFile.close()

                    if (result):
                        if (lengthOfYearUrlDictionary > 0):
                            print("...creating title slide")
                            createTitleSlide(url, beginYear, endYear,
                                             mementoGIFsPath)
                            print(
                                "...labelling screenshots and converting to gif"
                            )
                            convertToAnimatedGIF(mementoGIFsPath)
                            print(
                                "...done labelling screenshots and converting to gif"
                            )
                            print("...optimizing Gifs")
                            optimizeGifs(mementoGIFsPath)
                            print("...done optimizing Gifs")
                            print("...creating mp4 file")
                            generateMP4(mementoGIFsPath, musicTrack, startTime)
                            print("...done creating mp4 file")
                        '''
						#this block has been deprecated - start
						print "...prepending " + globalDataFileName + " with url: ", url

						tumblrDataFile = open(globalDataFileName, 'r')
						tumblrDataFileLines = tumblrDataFile.readlines()
						tumblrDataFile.close()

						tumblrDataFile = open(globalDataFileName, 'w')

						if( len(screen_name)>0 and len(tweetID)>0 ):
							screen_name = screen_name.strip()
							tweetID = tweetID.strip()
							tumblrDataFile.write(url + "," + " [tumblrUploadIDPlaceHolder], " + screen_name + ", " + tweetID + "\n")
						else:
							tumblrDataFile.write(url + "," + " [tumblrUploadIDPlaceHolder]\n")


						tumblrDataFile.writelines(tumblrDataFileLines)
						tumblrDataFile.close()
						print "...done appending " + globalDataFileName + " with url"
						#this block has been deprecated - end
						'''
                    else:
                        print('...deleting empty bad result:',
                              possibleFolderNameToDelete)
                        #someThingWentWrongFlag = True could mean that the request to the server was not successful,
                        #but could be successful in the future
                        someThingWentWrongFlag = True
                        co = 'rm -r ' + possibleFolderNameToDelete

                        subprocess.getoutput(co)
                else:
                    print('...deleting empty bad result:',
                          possibleFolderNameToDelete)
                    #someThingWentWrongFlag = True could mean that the request to the server was not successful,
                    #but could be successful in the future
                    someThingWentWrongFlag = True
                    co = 'rm -r ' + possibleFolderNameToDelete

                    subprocess.getoutput(co)

            except:
                exc_type, exc_obj, exc_tb = sys.exc_info()
                fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
                print((fname, exc_tb.tb_lineno, sys.exc_info()))
                #urlsFile.close()
                #tumblrDataFile.close()

                errorMessage = (fname, exc_tb.tb_lineno, sys.exc_info())
                #mod1
                sendErrorEmail(str(errorMessage))
    else:
        print("Url length error: Url length must be greater than zero")

    return someThingWentWrongFlag
Exemplo n.º 2
0
def timelapse(url, screen_name='', tweetID=''):

    if (len(url) > 0):
        url = url.lower()
        mementoGIFsPath, folderAlreadyExists = generateFolderNameFromURLUsingHash(
            url)

        #if folder exists exit - start

        if (folderAlreadyExists):
            print '... folder already exists, exiting'
            return

        #if folder exists exit - end

        if (len(mementoGIFsPath) > 0):

            try:

                possibleFolderNameToDelete = os.getcwd(
                ) + '/' + mementoGIFsPath + '/'

                print "...opening urlsFile.txt"
                urlsFile = open("./" + mementoGIFsPath + "/urlsFile.txt", "w")

                print "...getting memento pages"
                pages = getMementosPages(url)
                print "...done getting memento pages"

                mementosList = []
                for i in range(0, len(pages)):
                    mementos = getItemGivenSignature(pages[i])
                    mementosList.append(mementos)

                yearUrlDictionary = get1MementoPerYear(
                    mementosList, globalMementoUrlDateTimeDelimeter)

                #sort by date
                #this does not seem to return dictionary, list instead
                sortedKeysOfYearUrlDictionary = sorted(
                    yearUrlDictionary.keys())

                lengthOfYearUrlDictionary = len(yearUrlDictionary)
                if (lengthOfYearUrlDictionary > 0):

                    beginYear = str(sortedKeysOfYearUrlDictionary[0])
                    endYear = str(sortedKeysOfYearUrlDictionary[
                        len(sortedKeysOfYearUrlDictionary) - 1])

                    urlsFile.write("What Did " + url + " Look Like From " +
                                   beginYear + " To " + endYear + "?\n\n")
                    urlsFile.write("Links" + ":\n")

                #print len(yearUrlDictionary), " years "

                #for year,url in yearUrlDictionary.items():
                #       print year, ",", url

                print "...taking screenshots"
                result = takeScreenshots(yearUrlDictionary, mementoGIFsPath,
                                         urlsFile)
                print "...done taking screenshots"

                print "...done opening urlsFile.txt"
                urlsFile.close()

                if (result):
                    print "...labelling screenshots and converting to gif"
                    convertToAnimatedGIF(mementoGIFsPath)
                    print "...done labelling screenshots and converting to gif"
                    print "...optimizing Gifs"
                    optimizeGifs(mementoGIFsPath)
                    print "...done optimizing Gifs"
                    '''
                                        #this block has been deprecated - start
                                        print "...prepending " + globalDataFileName + " with url: ", url
                                        tumblrDataFile = open(globalDataFileName, 'r')
                                        tumblrDataFileLines = tumblrDataFile.readlines()
                                        tumblrDataFile.close()
                                        tumblrDataFile = open(globalDataFileName, 'w')
                                        if( len(screen_name)>0 and len(tweetID)>0 ):
                                                screen_name = screen_name.strip()
                                                tweetID = tweetID.strip()
                                                tumblrDataFile.write(url + "," + " [tumblrUploadIDPlaceHolder], " + screen_name + ", " + tweetID + "\n")
                                        else:
                                                tumblrDataFile.write(url + "," + " [tumblrUploadIDPlaceHolder]\n")
                                        tumblrDataFile.writelines(tumblrDataFileLines)
                                        tumblrDataFile.close()
                                        print "...done appending " + globalDataFileName + " with url"
                                        #this block has been deprecated - end
                                        '''
                else:
                    print '...deleting empty bad result:', possibleFolderNameToDelete
                    co = 'rm -r ' + possibleFolderNameToDelete

                    commands.getoutput(co)

            except:
                exc_type, exc_obj, exc_tb = sys.exc_info()
                fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
                print(fname, exc_tb.tb_lineno, sys.exc_info())
                #urlsFile.close()
                #tumblrDataFile.close()

                errorMessage = (fname, exc_tb.tb_lineno, sys.exc_info())
                #mod1
                sendErrorEmail(str(errorMessage))
    else:
        print "Url length error: Url length must be greater than zero"
Exemplo n.º 3
0
def makeStatusUpdateAndNotifyReferrer(twitterStatusUpdateMessage,
                                      screen_nameOfUserWhoSuggestedUri,
                                      tweet_id, URL, link, instagramLink,
                                      filename):

    modifyEntryFlag = False
    twitterStatusUpdateMessage = twitterStatusUpdateMessage.strip()
    if (len(twitterStatusUpdateMessage) > 0):
        try:

            print('...status update:',
                  twitterStatusUpdateMessage.strip() + '\n#memento')
            #MOD
            updateStatusWithMedia(
                statusUpdateString=twitterStatusUpdateMessage.strip() +
                '\n#memento',
                filename=filename)

            modifyEntryFlag = True
        except:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
            print((fname, exc_tb.tb_lineno, sys.exc_info()))

            debugOutputFile = open(debugOutputFileName, 'w')
            debugOutputFile.write('status update message: ' +
                                  twitterStatusUpdateMessage + '\n')
            debugOutputFile.write(fname + ', ' + str(exc_tb.tb_lineno) + ', ' +
                                  str(sys.exc_info()) + '\n')

            errorMessage = (fname, exc_tb.tb_lineno, sys.exc_info())
            sendErrorEmail(str(errorMessage))

        print("...sending " + screen_nameOfUserWhoSuggestedUri +
              " direct message")

    screen_nameOfUserWhoSuggestedUri = screen_nameOfUserWhoSuggestedUri.strip()
    tweet_id = tweet_id.strip()
    URL = URL.strip()
    link = link.strip()
    if (len(screen_nameOfUserWhoSuggestedUri) > 0 and len(tweet_id) > 0
            and len(URL) > 0 and len(link) > 0):
        try:

            #sendSomeoneADirectMessage(screen_nameOfUserWhoSuggestedUri, 'Hello ' + screen_nameOfUserWhoSuggestedUri + ',\n' + getCanonicalUrl(URL) + ' has been posted on http://bit.ly/whatdiditlooklike')
            #send a message under the same thread

            pageTitle = getPageTitle('http://' + getCanonicalUrl(URL))

            if (len(pageTitle) > 40):
                pageTitle = pageTitle[0:39]
                pageTitle = pageTitle + '...'

            notificationMessage = ' ' + getCanonicalUrl(
                URL
            ) + ' (' + pageTitle + ') has been posted: ' + link + '\n' + instagramLink

            print('...status notification', notificationMessage)
            #MOD
            updateStatusWithMedia(statusUpdateString=notificationMessage,
                                  filename=filename,
                                  tweet_id=tweet_id,
                                  screen_name=screen_nameOfUserWhoSuggestedUri)
            modifyEntryFlag = True
        except:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
            print((fname, exc_tb.tb_lineno, sys.exc_info()))

            debugOutputFile = open(debugOutputFileName, 'w')
            debugOutputFile.write('status update message: ' +
                                  twitterStatusUpdateMessage + '\n')
            debugOutputFile.write(fname + ', ' + str(exc_tb.tb_lineno) + ', ' +
                                  str(sys.exc_info()) + '\n')

            errorMessage = (fname, exc_tb.tb_lineno, sys.exc_info())
            sendErrorEmail(str(errorMessage))

    return modifyEntryFlag
Exemplo n.º 4
0
def postToTumblrQueue():

    print('...post to queue called')

    #get new items to post:
    nominationTuples = []
    try:
        nominationsFile = open(globalDataFileName, "r")
        nominationTuples = nominationsFile.readlines()
        nominationsFile.close()
    except:
        exc_type, exc_obj, exc_tb = sys.exc_info()
        fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        print((fname, exc_tb.tb_lineno, sys.exc_info()))
        nominationsFile.close()
        return

    print("read lines: ", len(nominationTuples))

    if (len(nominationTuples) > 0):

        #<index, [int(POST-ID), str(STATUS-UPDATE-WHEN-PUBLIC)]>
        dictionaryOfLinesToModify = {}

        for i in range(0, len(nominationTuples)):
            nominationData = nominationTuples[i].split(' <> ')
            print('...nominationData:', nominationData)
            '''
			0: URL
			1: SCREEN-NAME
			2: DATE-TIME
			3: TWEET-ID
			(optional 
				4: POST-ID
				5: STATUS-UPDATE-WHEN-PUBLIC
				6: POST-FLAG
			'''
            #this url has not been posted, so post to queue
            if (len(nominationData) < 6):

                URL = nominationData[0].strip()
                screen_name = nominationData[1].strip()
                tweetDateTimeString = nominationData[2].strip()
                tweetID = nominationData[3].strip()
                dateRange = nominationData[4].strip()

                canonicalURL = getCanonicalUrl(URL)
                folderName = ''

                #check if this post has been made already - start

                print('...checking if post has been made already')

                #CAUTION - start
                #check not required since timelapseTwitter.py runs before timelapseSubEngine.py
                #and does the isThisURLWithinNominationDifferential() check
                #goAheadFlag, dateDiff = isThisURLWithinNominationDifferential(URL, tweetDateTimeString)
                goAheadFlag = True
                #CAUTION - end

                #check if this post has been made already - end

                formattedTag = getFormattedTagURL(canonicalURL)
                #this URL is not public
                if (goAheadFlag):
                    #debug - start
                    #uploadAnimatedGifToTumblr(URL)

                    #folderName is hash
                    folderName = getHashString(canonicalURL)
                    print('...hashFolderName:', folderName)
                    postID, beginYear, endYear, instagramLink = uploadAnimatedGifToSocialMedia(
                        folderName, URL)

                    #print 'folderName', folderName
                    #postID = 123
                    #beginYear = '1997'
                    #endYear = '2014'
                    #debug - end

                    #if post is successful, get a status update message - start
                    #TAG-MODIFICATION-ZONE
                    link = globalBlogName + '/tagged/' + formattedTag
                    pageTitle = getPageTitle('http://' + canonicalURL)
                    pageTitle = pageTitle.strip()

                    #folderNameUrl - 22
                    #link - 22
                    #beginYear - 4
                    #endYear - 4
                    #pageTitle - variable
                    #\n#memento - 9
                    #max pageTitle is 42
                    #memento 8
                    if (len(pageTitle) > 40):
                        pageTitle = pageTitle[0:39]
                        pageTitle = pageTitle + '...'
                    twitterStatusUpdateMessage = getRandomStatusUpdateMessage(
                        canonicalURL, pageTitle, beginYear, endYear,
                        link + ", " + instagramLink)
                    print('...status update message when public: ',
                          twitterStatusUpdateMessage)

                else:
                    #this URL is already public probably due to TME, so notify referrer and trick notify notifyOnPostApproved() by adding public so it will skip this entry
                    twitterStatusUpdateMessage = 'STATUS UPDATE MESSAGE <> PUBLIC'
                    beginYear = '0000'
                    endYear = '0000'
                    postID = '0000'
                    instagramLink = '0000'

                    #TAG-MODIFICATION-ZONE
                    notificationMessage = ' Your request (' + tweetDateTimeString + ') has already been posted. Please retry in ' + str(
                        dateDiff
                    ) + ' days: ' + globalBlogName + '/tagged/' + formattedTag
                    #MOD
                    updateStatus(statusUpdateString=notificationMessage,
                                 screen_name=screen_name,
                                 tweet_id=tweetID)

                #if post is successful, get a status update message - end
                if (len(twitterStatusUpdateMessage) > 0 and len(folderName) > 0
                        and len(beginYear) > 0 and len(endYear) > 0
                        and postID > 0 and len(instagramLink) > 0):
                    dictionaryOfLinesToModify[i] = [
                        postID, instagramLink, twitterStatusUpdateMessage
                    ]

        #if a new post has been placed on the queue modify entry line in input file
        if (len(dictionaryOfLinesToModify) > 0):

            #modify input lines with new data due to post on queue

            for index, postDataArray in list(
                    dictionaryOfLinesToModify.items()):
                postID = str(postDataArray[0])
                instaLink = postDataArray[1]
                twitterStatusUpdateMessageWhenPublic = postDataArray[2]
                nominationTuples[index] = nominationTuples[index].strip(
                ) + ' <> ' + postID + ' <> ' + instaLink + ' <> ' + twitterStatusUpdateMessageWhenPublic + '\n'

            try:
                nominationsFile = open(globalDataFileName, 'w')
                nominationsFile.writelines(nominationTuples)
                nominationsFile.close()

                #send notification email - start
                sendErrorEmail('wdill - new items in queue',
                               'Queue Event Notification')
                #send notification email - end
            except:
                exc_type, exc_obj, exc_tb = sys.exc_info()
                fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
                print((fname, exc_tb.tb_lineno, sys.exc_info()))
                nominationsFile.close()
                return
Exemplo n.º 5
0
def getRequestUrls():

	sinceIDValue = ''
	sinceIDFilename = globalPrefix + 'sinceID.txt'
	try:
		print 'f:', sinceIDFilename

		sinceIDFile = open(sinceIDFilename, 'r')
		prevSinceIDFile = open(globalPrefix + 'prev_sinceID.txt', 'w')

		line = sinceIDFile.readline()

		if(len(line) > 1):
			sinceIDValue = long(line)
		else:
			sinceIDValue = long('0')

		prevSinceIDFile.write(str(sinceIDValue) + '\n')

		sinceIDFile.close()
		prevSinceIDFile.close()
	except:
		exc_type, exc_obj, exc_tb = sys.exc_info()
		fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
		print(fname, exc_tb.tb_lineno, sys.exc_info() )
		sinceIDValue = long('0')


	#get spam filter coeff.
	spamFilterCoeff = getConfigParameters('spamFilterCoeff')
	print 'spamFilterCoeff', spamFilterCoeff

	requestsRemaining = 0
	try:
		requestsRemaining = api.rate_limit_status()['resources']['search']['/search/tweets']['remaining']
	except:
		requestsRemaining = 0
		#m1-handle exception
		exc_type, exc_obj, exc_tb = sys.exc_info()
		fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
		print(fname, exc_tb.tb_lineno, sys.exc_info() )

		errorMessage = (fname, exc_tb.tb_lineno, sys.exc_info() )
		sendErrorEmail( str(errorMessage) )


	#requestsRemaining = 10
	print "Before Request remaining: ", requestsRemaining


	if( requestsRemaining > 0 ):
		#<user, expandedUrlsArray>
		twitterUsersDict = {}


		#assume initially tweet is present
		isTweetPresentFlag = True
		while( isTweetPresentFlag ):
			
			#if tweet is present this will change the False to True, else it will remain False and Stop the loop
			isTweetPresentFlag = False
			print "current sinceIDValue: ", sinceIDValue
			#m1-handle exception
			try:
				for tweet in tweepy.Cursor(api.search, q="%23icanhazmemento", since_id=sinceIDValue).items(30):
					print
					isTweetPresentFlag = True

					# From 2015-07-12 18:45:11
					# To   Sun, 12 Jul 2015 14:45:11 GMT
					localTweetDatetime = datetime_from_utc_to_local(tweet.created_at)
					#localTweetDatetime = tweet.created_at
					localTweetDatetime = localTweetDatetime.strftime('%a, %d %b %Y %H:%M:%S')
					localTweetDatetime = str(localTweetDatetime) + ' GMT'


					#update since_id
					if( tweet.id > sinceIDValue ):
						sinceIDValue = tweet.id

					#MOD
					print localTweetDatetime, ",tweet_id:", tweet.id, ",", tweet.user.screen_name, " - ", tweet.text

					#get urls from tweet - start
					#since even though access to none short url, still meant that
					#url has to be chased down until the final value, no need to access none
					#short url
					shortTwitterUrls = []
					if( tweet.text.find('#icanhazmemento') != -1 ):
						for shortURL in tweet.entities['urls']:
							#print 'n: ', shortURL['expanded_url']
							shortTwitterUrls.append(shortURL['url'])
					#get urls from tweet - end


					#if this tweet is in response to a parent tweet with link(s) - start
					if( tweet.in_reply_to_status_id is not None and len(shortTwitterUrls) == 0):
						print 'parent ID:', tweet.in_reply_to_status_id

						parentTweet = api.get_status(tweet.in_reply_to_status_id)
						print 'parent tweet:', parentTweet.text
						for shortURL in parentTweet.entities['urls']:
							#print 'n: ', shortURL['expanded_url']
							shortTwitterUrls.append(shortURL['url'])
					#if this tweet is in response to a parent tweet with link(s) - end

					if(len(shortTwitterUrls) != 0):
						for url in shortTwitterUrls:
							potentialExpandedUrl = expandUrl(url)

							if( len(potentialExpandedUrl) > 0 ):

								#url normalization - start
								if( potentialExpandedUrl[-1] == '/' ):
									potentialExpandedUrl = potentialExpandedUrl[:-1]
								#url normalization - end

								#create new entry for user since user is not in dictionary
								print '...potentialExpandedUrl:', potentialExpandedUrl
								potentialExpandedUrl = potentialExpandedUrl.strip()

								#note spam filter not implemented since twitter blocks spam
								twitterUsersDict[tweet.id] = {}
								twitterUsersDict[tweet.id]['screen_name'] = tweet.user.screen_name
								twitterUsersDict[tweet.id]['expandedUrl'] = potentialExpandedUrl
								twitterUsersDict[tweet.id]['create_datetime'] = localTweetDatetime
						
								'''
								#faulty logic
								if( tweet.user.screen_name in twitterUsersDict):
									
									
									#spam filter measure - start
									if( len(twitterUsersDict[tweet.user.screen_name]) < spamFilterCoeff ):
										twitterUsersDict[tweet.user.screen_name]['potentialExpandedUrlsArray'].append(potentialExpandedUrl)
									#spam filter measure - end
								else:
									#twitterUsersDict[tweet.user.screen_name] = [potentialExpandedUrl]

									twitterUsersDict[tweet.user.screen_name] = {}

									twitterUsersDict[tweet.user.screen_name]['potentialExpandedUrlsArray'] = [potentialExpandedUrl]
									twitterUsersDict[tweet.user.screen_name]['create_datetime'] = localTweetDatetime
									twitterUsersDict[tweet.user.screen_name]['tweet_id'] = tweet.id
								'''
			except:
				exc_type, exc_obj, exc_tb = sys.exc_info()
				fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
				print(fname, exc_tb.tb_lineno, sys.exc_info() )

				errorMessage = (fname, exc_tb.tb_lineno, sys.exc_info() )
				sendErrorEmail( str(errorMessage) )
			
			if( isTweetPresentFlag ):
				print '...sleeping for 15 seconds'
				time.sleep(15)
	try:
		#MOD
		sinceIDFile = open(sinceIDFilename, 'w')
		
		#print 'DEBUG CAUTION, sinceIDValue SET'
		#print
		#sinceIDValue = 630131997084622848

		sinceIDFile.write(str(sinceIDValue) + '\n')
		sinceIDFile.close()
		
	except:
		exc_type, exc_obj, exc_tb = sys.exc_info()
		fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
		print(fname, exc_tb.tb_lineno, sys.exc_info() )

	return twitterUsersDict
Exemplo n.º 6
0
def extractRequestsFromTwitter():

    print("...getting potential requests from twitter")
    expandedUrlsDictionary = getRequestUrls()

    ###################################---------------TESTING-START---------------###################################
    #return
    ###################################---------------TESTING-END-----------------###################################

    #send tweet.user.screen_name that post has been made
    #cron job to retrieve this tweets

    if (len(expandedUrlsDictionary) > 0):

        #listOfAlreadyNotifiedTweetIDs = []
        goAheadFlag = False
        requestsFile = open(globalRequestFilename, "a")
        #send notifications
        for screen_name, urls in expandedUrlsDictionary.items():

            for u in urls:

                #globalDictionaryOfTweetExtraInformation[u][0] = tweet.id
                #globalDictionaryOfTweetExtraInformation[u][1] = created_at_datetime

                print()
                print(u)

                #check if this post has been made already - start
                print('...checking if post has been made already')
                goAheadFlag, dateDiff = isThisURLWithinNominationDifferential(
                    u, str(globalDictionaryOfTweetExtraInformation[u][1]))
                #check if this post has been made already - end

                #get page title - start

                pageTitle = getPageTitle(u)

                if (len(pageTitle) > 37):
                    pageTitle = pageTitle[0:36]
                    pageTitle = pageTitle + '...'

                #get page title - end

                notificationMessage = ''
                dateTime = str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
                if (goAheadFlag):
                    print(
                        '...post has not been made, composing notification message'
                    )
                    notificationMessage = '@' + screen_name + ' Your request (' + u + ', ' + pageTitle + ') was received at ' + dateTime
                else:

                    #TAG-MODIFICATION-ZONE
                    print(
                        '...post HAS been made, composing notification message'
                    )
                    notificationMessage = '@' + screen_name + ' Your request (' + dateTime + ') has already been posted. Please retry in ' + str(
                        dateDiff
                    ) + ' days: ' + globalBlogName + '/tagged/' + getFormattedTagURL(
                        getCanonicalUrl(u))
                    #notify user post already exist and say when post can be renominated?

                try:
                    #send notification - start

                    #MOD
                    #print '...updating status ', notificationMessage
                    #print "...sending screen_name notification of receipt"
                    print('msg/u/id:', notificationMessage, u,
                          globalDictionaryOfTweetExtraInformation[u][0])
                    api.update_status(
                        notificationMessage,
                        globalDictionaryOfTweetExtraInformation[u][0])

                    #send notification - end

                    if (goAheadFlag):
                        print("...writing requests to file")

                        requestsFile.write(
                            u + ' <> ' + screen_name + ' <> ' +
                            str(globalDictionaryOfTweetExtraInformation[u]
                                [1]) + ' <> ' +
                            str(globalDictionaryOfTweetExtraInformation[u]
                                [0]) + ' <> ' +
                            str(globalDictionaryOfTweetExtraInformation[u]
                                [2]) + '\n')

                except:
                    print("")
                    exc_type, exc_obj, exc_tb = sys.exc_info()
                    fname = os.path.split(
                        exc_tb.tb_frame.f_code.co_filename)[1]
                    print((fname, exc_tb.tb_lineno, sys.exc_info()))

                    #MOD
                    errorMessage = (fname, exc_tb.tb_lineno, sys.exc_info())
                    sendErrorEmail(str(errorMessage))
        '''
        if( goAheadFlag ):
            #send email to me - start
            emailString = composeEmailString(expandedUrlsDictionary)

            #send email to inform me about new requests
            if(len(emailString) > 0):
                
                
                #sender = '*****@*****.**'
                #receivers = ['*****@*****.**','*****@*****.**']
                #message = 'this is a test message'
                #subject = 'test email service'
                #sendEmail(sender, receivers, subject, message)#

                sender = '*****@*****.**'
                receivers = ['*****@*****.**','*****@*****.**']
                message = 'twitter_requests_wdill.txt'

                subject = 'New requests to whatdiditlooklike'
                #MOD
                print "...sending email of requests"
                sendEmail(sender, receivers, subject, message)

            #send email to me - end
        '''

        requestsFile.close()