class Test_pipeline(unittest.TestCase):
def setUp(self):
self.p = Pipe()
self.p.size = 1
def test_add(self):
new_id = '12345'
self.p.add(new_id)
self.assertEqual(self.p.add(new_id), False)
def test_maintain(self):
for i in range(10):
self.p.add(i)
self.p.size
self.assertEqual(len(self.p.pipe), self.p.size)
class Tweet_talker(TwythonStreamer):
def __init__(self):
self.pipe = Pipe() #to check for duplicates
self.staging_file_path = s.STAGING_RAW_FEED_FILE_PATH
self.file_path = s.RAW_FEED_FILE_PATH
self.accepting_country_codes = s.TwitterStream_ACCEPTING_COUNTRY_CODES
self.file_name_prefix = s.twitter_raw_feed_file_prefix
self.last_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
self.last_file_handle = File_handler(self.last_full_path)
self.country_code_handle = Coordinate_handler()
super(Tweet_talker, self).__init__(s.twitter_consumer_key, s.twitter_consumer_secret,
s.twitter_oauth_key,s.twitter_oauth_secret)
def on_success(self, data):
if data:
if not self.country_code_handle.skip_this_data(data, self.accepting_country_codes) \
and self.pipe.add(data.get("id")):
staging_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
if (self.last_full_path == staging_full_path):
handle_to_use = self.last_file_handle
else:
# rollover file and copy over the old file to permanant location
handle_to_use = File_handler(staging_full_path)
handle_to_use.copy_file_to(self.last_full_path, self.file_path)
#reset last path and handle
self.last_full_path = staging_full_path
self.last_file_handle = handle_to_use
self.last_file_handle.append_to_file_as_json(data)
def on_error(self, status_code, data):
print "Error code:{0}, Message:{1}".format(status_code, data)
self.disconnect()
def __init__(self):
self.pipe = Pipe() #to check for duplicates
self.staging_file_path = s.STAGING_RAW_FEED_FILE_PATH
self.file_path = s.RAW_FEED_FILE_PATH
self.accepting_country_codes = s.TwitterStream_ACCEPTING_COUNTRY_CODES
self.file_name_prefix = s.twitter_raw_feed_file_prefix
self.last_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
self.last_file_handle = File_handler(self.last_full_path)
self.country_code_handle = Coordinate_handler()
super(Tweet_talker, self).__init__(s.twitter_consumer_key, s.twitter_consumer_secret,
s.twitter_oauth_key,s.twitter_oauth_secret)
def setUp(self):
self.p = Pipe()
self.p.size = 1
