def onTerminateFunc():
def doAction():
del FollowerExtractorGateThread.follower_extractor_threads[
twitterSession]
criticalSection(
FollowerExtractorGateThread.
_follower_extractor_threads_lock, doAction)
def _run(self):
while True:
copy = criticalSection(self.instances._lock,
lambda: dict(self.instances._by_oauth))
for oauth, instance in copy.iteritems():
assert isinstance(instance, TwitterInstance)
if instance.enable_shutdown_after_no_usage and instance.age > self.max_inactive:
# I want to see this in the error log.
logger.critical(
'Cleaning up instance with oauth: %s, it has been inactive for > %dms'
% (unicode(oauth), self.max_inactive))
self.instances.removeTwitterInstanceByAuth(oauth)
if instance.construct_age > self.max_construct_age:
logger.critical(
'Restarting instance with oauth: %s, it has been alive > %dms'
% (unicode(oauth), self.max_construct_age))
result = restartTwitterInstanceByAuth(
self.instances, oauth)
if result is not None:
logger.error(
'Failed to restart instance with oauth: %s, reason: %s'
% (unicode(oauth), result))
time.sleep(2)
def getUniqueInstanceKey(self):
def func():
instanceKey = unicode(getUniqueId())
while instanceKey in self._by_instance_key:
instanceKey = unicode(getUniqueId())
return instanceKey
return criticalSection(self._lock, func)
def getUniqueInstanceKey(self):
def func():
instanceKey = unicode(getUniqueId())
while instanceKey in self._by_instance_key:
instanceKey = unicode(getUniqueId())
return instanceKey
return criticalSection(self._lock, func)
def updateGroupCache(self, newValue):
if not self.isGroupCacheEnabled:
return newValue
def doChange():
self.signalDataPtr = newValue
return newValue
return criticalSection(self._signalDataPtrLock, doChange)
def createInstance(self, twitterAuthentication, geographic_setup_string,
keywords, instance_setup_code):
def func():
twitterInstance = TwitterInstance(self.getUniqueInstanceKey(),
self, twitterAuthentication,
geographic_setup_string,
keywords, instance_setup_code)
return twitterInstance
return criticalSection(self._lock, func)
def createInstance(self, twitterAuthentication, geographic_setup_string, keywords, instance_setup_code):
def func():
twitterInstance = TwitterInstance(self.getUniqueInstanceKey(),
self,
twitterAuthentication,
geographic_setup_string,
keywords,
instance_setup_code)
return twitterInstance
return criticalSection(self._lock, func)
def getExtractorThreadByTwitterSession(self, twitterSession):
if not twitterSession.is_session_active:
return None
extractorThread = criticalSection(
FollowerExtractorGateThread._follower_extractor_threads_lock,
lambda: FollowerExtractorGateThread.follower_extractor_threads.get(
twitterSession, None))
if extractorThread is not None:
return extractorThread
else:
def onTerminateFunc():
def doAction():
del FollowerExtractorGateThread.follower_extractor_threads[
twitterSession]
criticalSection(
FollowerExtractorGateThread.
_follower_extractor_threads_lock, doAction)
newThread = FollowerExtractorThread(
self.geocode_user_config,
outputQueue=self.output_queue,
twitterSession=twitterSession,
onTerminateFunc=onTerminateFunc,
userAnalysisList=self.user_analysis_list)
def doAction():
FollowerExtractorGateThread.follower_extractor_threads[
twitterSession] = newThread
criticalSection(
FollowerExtractorGateThread._follower_extractor_threads_lock,
doAction)
newThread.start()
return newThread
def _run(self):
while True:
copy = criticalSection(self.instances._lock, lambda: dict(self.instances._by_oauth))
for oauth, instance in copy.iteritems():
assert isinstance(instance, TwitterInstance)
if instance.enable_shutdown_after_no_usage and instance.age > self.max_inactive:
# I want to see this in the error log.
logger.critical('Cleaning up instance with oauth: %s, it has been inactive for > %dms' % (unicode(oauth), self.max_inactive))
self.instances.removeTwitterInstanceByAuth(oauth)
if instance.construct_age > self.max_construct_age:
logger.critical('Restarting instance with oauth: %s, it has been alive > %dms' % (unicode(oauth), self.max_construct_age))
result = restartTwitterInstanceByAuth(self.instances, oauth)
if result is not None:
logger.error('Failed to restart instance with oauth: %s, reason: %s' % (unicode(oauth),result))
time.sleep(2)
def getInstanceByAuth(self, oauth):
result = criticalSection(self._lock, lambda: self._by_oauth.get(oauth, None))
return result
def isAuthInUse(self, oauth):
return criticalSection(self._lock, lambda: oauth in self._by_oauth)
def isAuthInUse(self, oauth):
return criticalSection(self._lock, lambda: oauth in self._by_oauth)
def getInstanceByInstanceKey(self, instanceKey):
result = criticalSection(self._lock, lambda: self._by_instance_key.get(instanceKey, None))
return result
def getInstanceList(self):
return criticalSection(self._lock, lambda: list(self._by_instance_key.values()))
def isInstanceKeyInUse(self, instanceKey):
return criticalSection(self._lock, lambda: instanceKey in self._by_instance_key)
def getOriginalByFunction(self, value, hashFuncList=None, depth=0):
return criticalSection(self._lock, lambda: self.data.getOriginalByFunction(value, hashFuncList, depth))
def prune(self):
if self.prune_func is not None:
return criticalSection(self._lock, lambda: self.prune_func(dataStructure=self.data))
def getInstanceByAuth(self, oauth):
result = criticalSection(self._lock, lambda: self._by_oauth.get(oauth, None))
return result
def getOriginalByFunction(self, value, hashFuncList=None, depth=0):
return criticalSection(
self._lock, lambda: self.data.getOriginalByFunction(
value, hashFuncList, depth))
def _processSignal(self, signaler, data):
# It simplifies the implementation if signalers cannot be added while we process signals.
return criticalSection(self._signalers_lock,lambda: self.processSignal(signaler, data))
def isInstanceKeyInUse(self, instanceKey):
return criticalSection(self._lock, lambda: instanceKey in self._by_instance_key)
def getInstanceList(self):
return criticalSection(self._lock, lambda: list(self._by_instance_key.values()))
def getInstanceByInstanceKey(self, instanceKey):
result = criticalSection(self._lock, lambda: self._by_instance_key.get(instanceKey, None))
return result
def prune(self):
if self.prune_func is not None:
return criticalSection(
self._lock, lambda: self.prune_func(dataStructure=self.data))