def handle(self, *args, **options): me = singleton.SingleInstance(flavor_id="crontw") m = '%s: Will run the Twitter harvester' % now.strftime('%y-%m-%d %H:%M') print(m) twitterLogger.log(m) try: harvestTwitter() except: myEmailMessage[0] = 'TWITTER HARVEST ROUTINE HAS ENCOUNTERED A TOP-LEVEL ERROR:' twitterLogger.exception(myEmailMessage[0]) finally: print(myEmailTitle[0]) print(myEmailMessage[0]) twitterLogger.log("The harvest has ended for the Twitter harvesters",showTime=True) if not DEBUG: send_routine_email(myEmailTitle[0], myEmailMessage[0])
from Twitter.models import TWUser, follower, favorite_tweet, Tweet, get_from_any_or_create, MultipleObjectsReturned import psutil import time #from memory_profiler import profile from SocialNetworkHarvester_v1p0.settings import twitterLogger, DEBUG, LOG_DIRECTORY twitterLogger.reset_log() def log(*args, **kwargs): twitterLogger.log(*args, **kwargs) pretty = lambda s: twitterLogger.pretty(s) logerror = lambda s: twitterLogger.exception(s) process = psutil.Process() QUEUEMAXSIZE = 10000 threadsExitFlag = [False] updateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers updateQueue._name = 'updateQueue' friendsUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers friendsUpdateQueue._name = 'friendsUpdateQueue' followersUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers followersUpdateQueue._name = 'followersUpdateQueue' favoriteTweetUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers favoriteTweetUpdateQueue._name = 'favoriteTweetUpdateQueue'
import threading import queue from datetime import datetime, timedelta from django.utils.timezone import utc from Twitter.models import TWUser, follower, favorite_tweet, Tweet, get_from_any_or_create import psutil #from memory_profiler import profile from SocialNetworkHarvester_v1p0.settings import twitterLogger, DEBUG, LOG_DIRECTORY def log(*args, **kwargs): twitterLogger.log(*args, **kwargs) pretty = lambda s : twitterLogger.pretty(s) logerror = lambda s: twitterLogger.exception(s) QUEUEMAXSIZE = 0 threadsExitFlag = [False] updateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers updateQueue._name = 'updateQueue' friendsUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers friendsUpdateQueue._name = 'friendsUpdateQueue' followersUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers followersUpdateQueue._name = 'followersUpdateQueue' favoriteTweetUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers favoriteTweetUpdateQueue._name = 'favoriteTweetUpdateQueue' userHarvestQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers userHarvestQueue._name = 'userHarvestQueue' hashtagHarvestQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twHashtagHarvesters hashtagHarvestQueue._name = 'hashtagHarvestQueue'