示例#1
0
class Tweet_talker(TwythonStreamer):
    def __init__(self):
        self.pipe = Pipe() #to check for duplicates
        self.staging_file_path = s.STAGING_RAW_FEED_FILE_PATH
        self.file_path = s.RAW_FEED_FILE_PATH
        self.accepting_country_codes = s.TwitterStream_ACCEPTING_COUNTRY_CODES
        self.file_name_prefix = s.twitter_raw_feed_file_prefix
        
        self.last_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
                                                     Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
        self.last_file_handle = File_handler(self.last_full_path)
         
        self.country_code_handle = Coordinate_handler()        
        super(Tweet_talker, self).__init__(s.twitter_consumer_key, s.twitter_consumer_secret,
                                           s.twitter_oauth_key,s.twitter_oauth_secret)
    def on_success(self, data):
        if data:
            if not self.country_code_handle.skip_this_data(data, self.accepting_country_codes) \
                and self.pipe.add(data.get("id")):
                staging_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
                                                           Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
                if (self.last_full_path == staging_full_path):
                    handle_to_use = self.last_file_handle
                else:
                    # rollover file and copy over the old file to permanant location
                    handle_to_use = File_handler(staging_full_path)
                    handle_to_use.copy_file_to(self.last_full_path, self.file_path)
                    #reset last path and handle
                    self.last_full_path = staging_full_path
                    self.last_file_handle = handle_to_use
                self.last_file_handle.append_to_file_as_json(data)
    def on_error(self, status_code, data):
        print "Error code:{0}, Message:{1}".format(status_code, data)
        self.disconnect()
示例#2
0
 def on_success(self, data):
     if data:
         if not self.country_code_handle.skip_this_data(data, self.accepting_country_codes) \
             and self.pipe.add(data.get("id")):
             staging_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
                                                        Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
             if (self.last_full_path == staging_full_path):
                 handle_to_use = self.last_file_handle
             else:
                 # rollover file and copy over the old file to permanant location
                 handle_to_use = File_handler(staging_full_path)
                 handle_to_use.copy_file_to(self.last_full_path, self.file_path)
                 #reset last path and handle
                 self.last_full_path = staging_full_path
                 self.last_file_handle = handle_to_use
             self.last_file_handle.append_to_file_as_json(data)
示例#3
0
 def test_append_to_file_as_json(self):
     #append current time to file 
     self.f = File_handler(filename)
     data = {"key" : "this could have been any kind of data. Doesnt have to be dictionary"}
     self.f.append_to_file_as_json(data)
     
     with open(filename, 'r') as testfile:
         all_lines = testfile.readlines()
     os.remove(filename)
     self.assertEqual(data, json.loads(all_lines[len(all_lines)-1]))
示例#4
0
 def __init__(self):
     self.pipe = Pipe() #to check for duplicates
     self.staging_file_path = s.STAGING_RAW_FEED_FILE_PATH
     self.file_path = s.RAW_FEED_FILE_PATH
     self.accepting_country_codes = s.TwitterStream_ACCEPTING_COUNTRY_CODES
     self.file_name_prefix = s.twitter_raw_feed_file_prefix
     
     self.last_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
                                                  Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
     self.last_file_handle = File_handler(self.last_full_path)
      
     self.country_code_handle = Coordinate_handler()        
     super(Tweet_talker, self).__init__(s.twitter_consumer_key, s.twitter_consumer_secret,
                                        s.twitter_oauth_key,s.twitter_oauth_secret)
示例#5
0
class Test_utility(unittest.TestCase):
    def setUp(self):
        pass
    def tearDown(self):
        pass
    def test_get_current_utc_string(self):
        e = datetime.datetime.now()
        expected = e.strftime(settings.UTC_TIMESTAMP_FORMAT)
        actual = Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT)
        self.assertEqual(expected, actual)
    def test_append_to_file_as_json(self):
        #append current time to file 
        self.f = File_handler(filename)
        data = {"key" : "this could have been any kind of data. Doesnt have to be dictionary"}
        self.f.append_to_file_as_json(data)
        
        with open(filename, 'r') as testfile:
            all_lines = testfile.readlines()
        os.remove(filename)
        self.assertEqual(data, json.loads(all_lines[len(all_lines)-1]))
    def test_skip_this_data(self):
        accepting_country_codes = settings.TwitterStream_ACCEPTING_COUNTRY_CODES
        
        f = Coordinate_handler()
        coord_mixture = {"SE": [[16.59929221,59.92538324]]}
        for key in coord_mixture:
            for coord in coord_mixture[key]:
                data = { "time" : str(time.time()), "coordinates": {"coordinates": coord}}
                self.assertEqual(f.skip_this_data(data, accepting_country_codes), (False if key in accepting_country_codes else True))
    def test_find_country_codes_for_coord(self):
        f = Coordinate_handler()
        coord_mixture = {"SE": [[16.59929221,59.92538324]]}
        for key in coord_mixture:
            for coord in coord_mixture[key]:
                # data = { "time" : str(time.time()), "coordinates": {"coordinates": coord}}
                self.assertEqual(f.find_country_code_for_coordinates(coord), key)