def run(self): while True: if self.stop_event.isSet() or self.crisis_stop_event.isSet(): break if self.scanner.is_empty(): sleep(1) else: try: since_time = get_utc(datetime.datetime.now()) for item in self.connect(): body = u"" if item.media.description.text is None: body = u"" else: body = item.media.description.text.decode('utf-8') if item.media.keywords.text is not None: body += item.media.keywords.text.decode('utf-8') send_dict = { 'since_time': since_time, 'item_id': unicode(item.id.text.rsplit('/', 1)[1]), 'body': body, 'creation_time': datetime.datetime.strptime( item.updated.text, "%Y-%m-%dT%H:%M:%S.000Z"), 'screen_name': item.author[0].name.text.decode('utf-8'), 'longitude': u"", 'latitude': u"", # item.geo.location() } self.to_filter(send_dict) sleep(self.settings['poll_delay']) except IOError: print statuses.fail( "The Youtube API failed.", sub= "Terminate the sourcer, check YouTube's settings, and try again." ) quit()
def __initialise_sentiment(self): filename = os.path.join(get_project_root(), 'backend/analysis/data/AFINN-111.txt') try: return_dict = dict( map(lambda (w, s): (w, int(s)), [ws.strip().split('\t') for ws in open(filename)])) except IOError: print statuses.fail( "Failed to find the specified sentiment data file.", sub=filename) return {} print statuses.success("Sentiment Analysis successfully initialised.") return return_dict
def __init__(self, check_all=True, analysis_manager=None): self.schema = indexer.index_exists('geo', indexer.Index_Types.Crisees) self.indexes = {} if self.schema: print statuses.success( "Geographical Analysis successfully initialised.") self.__reader = reader.Reader(self.schema, 'geo', indexer.Index_Types.Crisees) if check_all: from db.crisees.models import Event for event in Event.objects.all(): self.indexes[event.id] = self.check_event_index(event) else: print statuses.fail( "Failed to find the geographical places index.", "Geographical analysis will not work for this session.") self.__reader = None if analysis_manager is not None: self.verifier = Verifier(self, analysis_manager) self.verifier.start()
def __fail(self, source): self.__failed.append(source) print statuses.fail( ("The source %s has failed." % str(source)), sub = "This could be a failure to start, or another runtime problem.")
def __specific_invalid(self, source): self.__failed.append(source) print statuses.fail( ("The source %s has failed." % str(source)), sub = ("Check to see if %s has a valid specifics script." % str(source)))
def run(self): while True: if self.stop_event.isSet() or self.crisis_stop_event.isSet(): break if self.scanner.is_empty(): sleep(1) else: try: stream = self.connect() for tweet in stream: if self.crisis_stop_event.isSet(): stream.close quit() if self.stop_event.isSet() or self.scanner.has_changed(): stream.close() now = datetime.now() diff = now - self.last_stop print diff if diff.total_seconds() < 10: sleep(10) self.last_stop = now break if tweet.has_key('text'): user_id = unicode(tweet['user']['id_str']) screen_name = unicode(tweet['user']['screen_name']) creation_time = to_date(tweet['created_at']) location = unicode(tweet['user']['location']) time_zone = unicode(tweet['user']['time_zone']) longitude = u"" latitude = u"" if isinstance(tweet['geo'], dict) and 'type' in tweet['geo'] and tweet['geo']['type'].lower() == 'point': longitude = unicode(tweet['geo']['coordinates'][0]) latitude = unicode(tweet['geo']['coordinates'][1]) item_id = unicode(tweet['id']) body = unicode(tweet['text']) send_dict = { 'item_id' : item_id, 'body' : body, 'creation_time': creation_time, 'screen_name' : screen_name, 'longitude' : longitude, 'latitude' : latitude, 'user_id' : user_id, 'location' : location, 'time_zone' : time_zone, } self.to_filter(send_dict) except tweetstream.AuthenticationError: print statuses.fail( "Authentication for accessing the Twitter streaming API failed.", sub = "Terminate the backend, check Twitter's settings, and try again.") quit() except tweetstream.ConnectionError: print statuses.fail( "Connecting to the Twitter streaming API failed.", sub = ("After a grace period of %d seconds, I will try and connect again." % self.settings['connect_retry']),) sleep(self.settings['connect_retry'])