def run(self):
while True:
if self.stop_event.isSet() or self.crisis_stop_event.isSet():
break
if self.scanner.is_empty():
sleep(1)
else:
try:
since_time = get_utc(datetime.datetime.now())
for item in self.connect():
body = u""
if item.media.description.text is None:
body = u""
else:
body = item.media.description.text.decode('utf-8')
if item.media.keywords.text is not None:
body += item.media.keywords.text.decode('utf-8')
send_dict = {
'since_time':
since_time,
'item_id':
unicode(item.id.text.rsplit('/', 1)[1]),
'body':
body,
'creation_time':
datetime.datetime.strptime(
item.updated.text, "%Y-%m-%dT%H:%M:%S.000Z"),
'screen_name':
item.author[0].name.text.decode('utf-8'),
'longitude':
u"",
'latitude':
u"",
# item.geo.location()
}
self.to_filter(send_dict)
sleep(self.settings['poll_delay'])
except IOError:
print statuses.fail(
"The Youtube API failed.",
sub=
"Terminate the sourcer, check YouTube's settings, and try again."
)
quit()
def __initialise_sentiment(self):
filename = os.path.join(get_project_root(),
'backend/analysis/data/AFINN-111.txt')
try:
return_dict = dict(
map(lambda (w, s): (w, int(s)),
[ws.strip().split('\t') for ws in open(filename)]))
except IOError:
print statuses.fail(
"Failed to find the specified sentiment data file.",
sub=filename)
return {}
print statuses.success("Sentiment Analysis successfully initialised.")
return return_dict
def __init__(self, check_all=True, analysis_manager=None):
self.schema = indexer.index_exists('geo', indexer.Index_Types.Crisees)
self.indexes = {}
if self.schema:
print statuses.success(
"Geographical Analysis successfully initialised.")
self.__reader = reader.Reader(self.schema, 'geo',
indexer.Index_Types.Crisees)
if check_all:
from db.crisees.models import Event
for event in Event.objects.all():
self.indexes[event.id] = self.check_event_index(event)
else:
print statuses.fail(
"Failed to find the geographical places index.",
"Geographical analysis will not work for this session.")
self.__reader = None
if analysis_manager is not None:
self.verifier = Verifier(self, analysis_manager)
self.verifier.start()
def __fail(self, source):
self.__failed.append(source)
print statuses.fail(
("The source %s has failed." % str(source)),
sub = "This could be a failure to start, or another runtime problem.")
def __specific_invalid(self, source):
self.__failed.append(source)
print statuses.fail(
("The source %s has failed." % str(source)),
sub = ("Check to see if %s has a valid specifics script." % str(source)))
def run(self):
while True:
if self.stop_event.isSet() or self.crisis_stop_event.isSet():
break
if self.scanner.is_empty():
sleep(1)
else:
try:
stream = self.connect()
for tweet in stream:
if self.crisis_stop_event.isSet():
stream.close
quit()
if self.stop_event.isSet() or self.scanner.has_changed():
stream.close()
now = datetime.now()
diff = now - self.last_stop
print diff
if diff.total_seconds() < 10:
sleep(10)
self.last_stop = now
break
if tweet.has_key('text'):
user_id = unicode(tweet['user']['id_str'])
screen_name = unicode(tweet['user']['screen_name'])
creation_time = to_date(tweet['created_at'])
location = unicode(tweet['user']['location'])
time_zone = unicode(tweet['user']['time_zone'])
longitude = u""
latitude = u""
if isinstance(tweet['geo'], dict) and 'type' in tweet['geo'] and tweet['geo']['type'].lower() == 'point':
longitude = unicode(tweet['geo']['coordinates'][0])
latitude = unicode(tweet['geo']['coordinates'][1])
item_id = unicode(tweet['id'])
body = unicode(tweet['text'])
send_dict = {
'item_id' : item_id,
'body' : body,
'creation_time': creation_time,
'screen_name' : screen_name,
'longitude' : longitude,
'latitude' : latitude,
'user_id' : user_id,
'location' : location,
'time_zone' : time_zone,
}
self.to_filter(send_dict)
except tweetstream.AuthenticationError:
print statuses.fail(
"Authentication for accessing the Twitter streaming API failed.",
sub = "Terminate the backend, check Twitter's settings, and try again.")
quit()
except tweetstream.ConnectionError:
print statuses.fail(
"Connecting to the Twitter streaming API failed.",
sub = ("After a grace period of %d seconds, I will try and connect again." % self.settings['connect_retry']),)
sleep(self.settings['connect_retry'])