class Tweet_talker(TwythonStreamer):
def __init__(self):
self.pipe = Pipe() #to check for duplicates
self.staging_file_path = s.STAGING_RAW_FEED_FILE_PATH
self.file_path = s.RAW_FEED_FILE_PATH
self.accepting_country_codes = s.TwitterStream_ACCEPTING_COUNTRY_CODES
self.file_name_prefix = s.twitter_raw_feed_file_prefix
self.last_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
self.last_file_handle = File_handler(self.last_full_path)
self.country_code_handle = Coordinate_handler()
super(Tweet_talker, self).__init__(s.twitter_consumer_key, s.twitter_consumer_secret,
s.twitter_oauth_key,s.twitter_oauth_secret)
def on_success(self, data):
if data:
if not self.country_code_handle.skip_this_data(data, self.accepting_country_codes) \
and self.pipe.add(data.get("id")):
staging_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
if (self.last_full_path == staging_full_path):
handle_to_use = self.last_file_handle
else:
# rollover file and copy over the old file to permanant location
handle_to_use = File_handler(staging_full_path)
handle_to_use.copy_file_to(self.last_full_path, self.file_path)
#reset last path and handle
self.last_full_path = staging_full_path
self.last_file_handle = handle_to_use
self.last_file_handle.append_to_file_as_json(data)
def on_error(self, status_code, data):
print "Error code:{0}, Message:{1}".format(status_code, data)
self.disconnect()
def on_success(self, data):
if data:
if not self.country_code_handle.skip_this_data(data, self.accepting_country_codes) \
and self.pipe.add(data.get("id")):
staging_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
if (self.last_full_path == staging_full_path):
handle_to_use = self.last_file_handle
else:
# rollover file and copy over the old file to permanant location
handle_to_use = File_handler(staging_full_path)
handle_to_use.copy_file_to(self.last_full_path, self.file_path)
#reset last path and handle
self.last_full_path = staging_full_path
self.last_file_handle = handle_to_use
self.last_file_handle.append_to_file_as_json(data)
def test_append_to_file_as_json(self):
#append current time to file
self.f = File_handler(filename)
data = {"key" : "this could have been any kind of data. Doesnt have to be dictionary"}
self.f.append_to_file_as_json(data)
with open(filename, 'r') as testfile:
all_lines = testfile.readlines()
os.remove(filename)
self.assertEqual(data, json.loads(all_lines[len(all_lines)-1]))
def __init__(self):
self.pipe = Pipe() #to check for duplicates
self.staging_file_path = s.STAGING_RAW_FEED_FILE_PATH
self.file_path = s.RAW_FEED_FILE_PATH
self.accepting_country_codes = s.TwitterStream_ACCEPTING_COUNTRY_CODES
self.file_name_prefix = s.twitter_raw_feed_file_prefix
self.last_full_path = "{0}{1}{2}.txt".format(self.staging_file_path, self.file_name_prefix,
Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT))
self.last_file_handle = File_handler(self.last_full_path)
self.country_code_handle = Coordinate_handler()
super(Tweet_talker, self).__init__(s.twitter_consumer_key, s.twitter_consumer_secret,
s.twitter_oauth_key,s.twitter_oauth_secret)
class Test_utility(unittest.TestCase):
def setUp(self):
pass
def tearDown(self):
pass
def test_get_current_utc_string(self):
e = datetime.datetime.now()
expected = e.strftime(settings.UTC_TIMESTAMP_FORMAT)
actual = Date_handler().get_current_utc_date_string(settings.UTC_TIMESTAMP_FORMAT)
self.assertEqual(expected, actual)
def test_append_to_file_as_json(self):
#append current time to file
self.f = File_handler(filename)
data = {"key" : "this could have been any kind of data. Doesnt have to be dictionary"}
self.f.append_to_file_as_json(data)
with open(filename, 'r') as testfile:
all_lines = testfile.readlines()
os.remove(filename)
self.assertEqual(data, json.loads(all_lines[len(all_lines)-1]))
def test_skip_this_data(self):
accepting_country_codes = settings.TwitterStream_ACCEPTING_COUNTRY_CODES
f = Coordinate_handler()
coord_mixture = {"SE": [[16.59929221,59.92538324]]}
for key in coord_mixture:
for coord in coord_mixture[key]:
data = { "time" : str(time.time()), "coordinates": {"coordinates": coord}}
self.assertEqual(f.skip_this_data(data, accepting_country_codes), (False if key in accepting_country_codes else True))
def test_find_country_codes_for_coord(self):
f = Coordinate_handler()
coord_mixture = {"SE": [[16.59929221,59.92538324]]}
for key in coord_mixture:
for coord in coord_mixture[key]:
# data = { "time" : str(time.time()), "coordinates": {"coordinates": coord}}
self.assertEqual(f.find_country_code_for_coordinates(coord), key)