コード例 #1
0
ファイル: test_common.py プロジェクト: shikarkhane/fetcher
 def test_find_country_codes_for_coord(self):
     f = Coordinate_handler()
     coord_mixture = {"SE": [[16.59929221,59.92538324]]}
     for key in coord_mixture:
         for coord in coord_mixture[key]:
             # data = { "time" : str(time.time()), "coordinates": {"coordinates": coord}}
             self.assertEqual(f.find_country_code_for_coordinates(coord), key)
コード例 #2
0
ファイル: test_common.py プロジェクト: shikarkhane/fetcher
 def test_skip_this_data(self):
     accepting_country_codes = settings.TwitterStream_ACCEPTING_COUNTRY_CODES
     
     f = Coordinate_handler()
     coord_mixture = {"SE": [[16.59929221,59.92538324]]}
     for key in coord_mixture:
         for coord in coord_mixture[key]:
             data = { "time" : str(time.time()), "coordinates": {"coordinates": coord}}
             self.assertEqual(f.skip_this_data(data, accepting_country_codes), (False if key in accepting_country_codes else True))
コード例 #3
0
ファイル: tweet_talker.py プロジェクト: shikarkhane/fetcher
class Tweet_talker(TwythonStreamer):
    def __init__(self):
        self.pipe = Pipe() #to check for duplicates
        self.staging_file_path = s.STAGING_RAW_FEED_FILE_PATH
        self.file_path = s.RAW_FEED_FILE_PATH
        self.accepting_country_codes = s.TwitterStream_ACCEPTING_COUNTRY_CODES
        self.file_name_prefix = s.twitter_raw_feed_file_prefix
        
        self.last_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
                                                     Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
        self.last_file_handle = File_handler(self.last_full_path)
         
        self.country_code_handle = Coordinate_handler()        
        super(Tweet_talker, self).__init__(s.twitter_consumer_key, s.twitter_consumer_secret,
                                           s.twitter_oauth_key,s.twitter_oauth_secret)
    def on_success(self, data):
        if data:
            if not self.country_code_handle.skip_this_data(data, self.accepting_country_codes) \
                and self.pipe.add(data.get("id")):
                staging_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
                                                           Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
                if (self.last_full_path == staging_full_path):
                    handle_to_use = self.last_file_handle
                else:
                    # rollover file and copy over the old file to permanant location
                    handle_to_use = File_handler(staging_full_path)
                    handle_to_use.copy_file_to(self.last_full_path, self.file_path)
                    #reset last path and handle
                    self.last_full_path = staging_full_path
                    self.last_file_handle = handle_to_use
                self.last_file_handle.append_to_file_as_json(data)
    def on_error(self, status_code, data):
        print "Error code:{0}, Message:{1}".format(status_code, data)
        self.disconnect()
コード例 #4
0
ファイル: tweet_talker.py プロジェクト: shikarkhane/fetcher
 def __init__(self):
     self.pipe = Pipe() #to check for duplicates
     self.staging_file_path = s.STAGING_RAW_FEED_FILE_PATH
     self.file_path = s.RAW_FEED_FILE_PATH
     self.accepting_country_codes = s.TwitterStream_ACCEPTING_COUNTRY_CODES
     self.file_name_prefix = s.twitter_raw_feed_file_prefix
     
     self.last_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
                                                  Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
     self.last_file_handle = File_handler(self.last_full_path)
      
     self.country_code_handle = Coordinate_handler()        
     super(Tweet_talker, self).__init__(s.twitter_consumer_key, s.twitter_consumer_secret,
                                        s.twitter_oauth_key,s.twitter_oauth_secret)