def handle(self, *args, **options):
me = singleton.SingleInstance(flavor_id="crontw")
m = '%s: Will run the Twitter harvester' % now.strftime('%y-%m-%d %H:%M')
print(m)
twitterLogger.log(m)
try:
harvestTwitter()
except:
myEmailMessage[0] = 'TWITTER HARVEST ROUTINE HAS ENCOUNTERED A TOP-LEVEL ERROR:'
twitterLogger.exception(myEmailMessage[0])
finally:
print(myEmailTitle[0])
print(myEmailMessage[0])
twitterLogger.log("The harvest has ended for the Twitter harvesters",showTime=True)
if not DEBUG:
send_routine_email(myEmailTitle[0], myEmailMessage[0])
from Twitter.models import TWUser, follower, favorite_tweet, Tweet, get_from_any_or_create, MultipleObjectsReturned
import psutil
import time
#from memory_profiler import profile
from SocialNetworkHarvester_v1p0.settings import twitterLogger, DEBUG, LOG_DIRECTORY
twitterLogger.reset_log()
def log(*args, **kwargs):
twitterLogger.log(*args, **kwargs)
pretty = lambda s: twitterLogger.pretty(s)
logerror = lambda s: twitterLogger.exception(s)
process = psutil.Process()
QUEUEMAXSIZE = 10000
threadsExitFlag = [False]
updateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
updateQueue._name = 'updateQueue'
friendsUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
friendsUpdateQueue._name = 'friendsUpdateQueue'
followersUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
followersUpdateQueue._name = 'followersUpdateQueue'
favoriteTweetUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
favoriteTweetUpdateQueue._name = 'favoriteTweetUpdateQueue'
import threading
import queue
from datetime import datetime, timedelta
from django.utils.timezone import utc
from Twitter.models import TWUser, follower, favorite_tweet, Tweet, get_from_any_or_create
import psutil
#from memory_profiler import profile
from SocialNetworkHarvester_v1p0.settings import twitterLogger, DEBUG, LOG_DIRECTORY
def log(*args, **kwargs):
twitterLogger.log(*args, **kwargs)
pretty = lambda s : twitterLogger.pretty(s)
logerror = lambda s: twitterLogger.exception(s)
QUEUEMAXSIZE = 0
threadsExitFlag = [False]
updateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
updateQueue._name = 'updateQueue'
friendsUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
friendsUpdateQueue._name = 'friendsUpdateQueue'
followersUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
followersUpdateQueue._name = 'followersUpdateQueue'
favoriteTweetUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
favoriteTweetUpdateQueue._name = 'favoriteTweetUpdateQueue'
userHarvestQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
userHarvestQueue._name = 'userHarvestQueue'
hashtagHarvestQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twHashtagHarvesters
hashtagHarvestQueue._name = 'hashtagHarvestQueue'
