from django.utils.timezone import utc
from Twitter.models import TWUser, follower, favorite_tweet, Tweet, get_from_any_or_create, MultipleObjectsReturned
import psutil
import time
#from memory_profiler import profile
from SocialNetworkHarvester_v1p0.settings import twitterLogger, DEBUG, LOG_DIRECTORY
twitterLogger.reset_log()
def log(*args, **kwargs):
twitterLogger.log(*args, **kwargs)
pretty = lambda s: twitterLogger.pretty(s)
logerror = lambda s: twitterLogger.exception(s)
process = psutil.Process()
QUEUEMAXSIZE = 10000
threadsExitFlag = [False]
updateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
updateQueue._name = 'updateQueue'
friendsUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
friendsUpdateQueue._name = 'friendsUpdateQueue'
followersUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
followersUpdateQueue._name = 'followersUpdateQueue'
favoriteTweetUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
from django.db import models
from django.db.utils import IntegrityError
from django.core.exceptions import MultipleObjectsReturned
import _mysql_exceptions
from SocialNetworkHarvester_v1p0.models import *
from django.utils.timezone import utc
import json
from datetime import datetime
import re
from SocialNetworkHarvester_v1p0.settings import twitterLogger, DEBUG
log = lambda s : twitterLogger.log(s) if DEBUG else 0
pretty = lambda s : twitterLogger.pretty(s) if DEBUG else 0
today = lambda : datetime.utcnow().replace(hour=0,minute=0,second=0,microsecond=0,tzinfo=utc)
############### PLACE ####################
class TWPlace(models.Model):
_ident = models.CharField(unique=True, max_length=255)
attributes = models.CharField(max_length=500, null=True)
bounding_box = models.CharField(max_length=500, null=True)
country = models.CharField(max_length=128, null=True)
full_name = models.CharField(max_length=255, null=True)
name = models.CharField(max_length=128, null=True)
place_type = models.CharField(max_length=128, null=True)
url = models.CharField(max_length=255, null=True)
def get_fields_description(self):
return {
"_ident": {
from django.db import models
from django.db.utils import IntegrityError
from django.core.exceptions import MultipleObjectsReturned
import _mysql_exceptions
from SocialNetworkHarvester_v1p0.models import *
from django.utils.timezone import utc
import json
from datetime import datetime
import re, time
from SocialNetworkHarvester_v1p0.settings import twitterLogger, DEBUG
log = lambda s : twitterLogger.log(s) if DEBUG else 0
pretty = lambda s : twitterLogger.pretty(s) if DEBUG else 0
today = lambda : datetime.utcnow().replace(hour=0,minute=0,second=0,microsecond=0,tzinfo=utc)
############### PLACE ####################
class TWPlace(models.Model):
_ident = models.CharField(unique=True, max_length=255)
attributes = models.CharField(max_length=500, null=True)
bounding_box = models.CharField(max_length=500, null=True)
country = models.CharField(max_length=128, null=True)
full_name = models.CharField(max_length=255, null=True)
name = models.CharField(max_length=128, null=True)
place_type = models.CharField(max_length=128, null=True)
url = models.CharField(max_length=255, null=True)
def get_fields_description(self):
return {
"_ident": {
import threading
import queue
from datetime import datetime, timedelta
from django.utils.timezone import utc
from Twitter.models import TWUser, follower, favorite_tweet, Tweet, get_from_any_or_create
import psutil
#from memory_profiler import profile
from SocialNetworkHarvester_v1p0.settings import twitterLogger, DEBUG, LOG_DIRECTORY
def log(*args, **kwargs):
twitterLogger.log(*args, **kwargs)
pretty = lambda s : twitterLogger.pretty(s)
logerror = lambda s: twitterLogger.exception(s)
QUEUEMAXSIZE = 0
threadsExitFlag = [False]
updateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
updateQueue._name = 'updateQueue'
friendsUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
friendsUpdateQueue._name = 'friendsUpdateQueue'
followersUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
followersUpdateQueue._name = 'followersUpdateQueue'
favoriteTweetUpdateQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
favoriteTweetUpdateQueue._name = 'favoriteTweetUpdateQueue'
userHarvestQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twUsers
userHarvestQueue._name = 'userHarvestQueue'
hashtagHarvestQueue = queue.Queue(maxsize=QUEUEMAXSIZE) #stores twHashtagHarvesters
hashtagHarvestQueue._name = 'hashtagHarvestQueue'