def setup(self): ''' Set up before each test. ''' # Set up the worker and its dependencies self.result_queue = MockResultQueue() self.task_queue = MockTaskQueue() self.tweet_factory = MockTweetFactory() self.worker = Worker(self.result_queue, self.task_queue, self.tweet_factory) self.worker_thread = Thread(target=self.worker.perform_tasks) self.worker_thread.is_daemon = False
class TestWorker(): ''' Unit tests for the Worker class. ''' def setup(self): ''' Set up before each test. ''' # Set up the worker and its dependencies self.result_queue = MockResultQueue() self.task_queue = MockTaskQueue() self.tweet_factory = MockTweetFactory() self.worker = Worker(self.result_queue, self.task_queue, self.tweet_factory) self.worker_thread = Thread(target=self.worker.perform_tasks) self.worker_thread.is_daemon = False def test_perform_tasks_count_tweets(self): ''' Tests the perform_tasks function when a count_tweets task is received. ''' # Load the test data self.task_queue.task = { 'job_id' : 'job_id', 'task' : 'count_tweets', 'coordinate_box' : {'min_lat' : 0, 'min_lon' : 0, 'max_lat' : 0, 'max_lon' : 0} } self.tweet_factory.tweets = ['tweet', 'tweet', 'tweet'] # Launch the worker thread self.worker_thread.start() # Wait a moment before shutting the thread down time.sleep(1) self.worker.shutdown() # Check the results assert self.result_queue.job_id == 'job_id', self.result_queue.job_id assert self.result_queue.coordinate_box == { 'min_lat' : 0, 'min_lon' : 0, 'max_lat' : 0, 'max_lon' : 0 }, self.result_queue.coordinate_box assert self.result_queue.count == 3, self.result_queue.count assert self.task_queue.finished_task is not None
def setup(self): ''' Set up before each test. ''' # Set up the testing configuration settings config = ConfigParser() config.add_section('worker') config.set('worker', 'batch_copy_size', '10') # Set up the mock testing components self.result_queue = MockResultQueue() self.task_queue = MockTaskQueue() self.data_factory = MockDataFactory() # Set up the Worker instance to be tested self.worker = Worker(config, self.result_queue, self.task_queue, self.data_factory)
from smcity.transformers.geojson_transformer import GeoJsonTransformer print "Loading the config settings..." config = ConfigParser() configFile = open("config/seattle_smcity.conf") config.readfp(configFile) configFile.close() # Load the GeoJSON description of the police beats police_beats_geojson = geojson.loads(open("seattle_slides/seattle_police_beats.geojson").read())["features"] # Set up the components data_factory = AwsDataFactory(config) result_queue = MockResultQueue() task_queue = MockTaskQueue() worker = Worker(config, result_queue, task_queue, data_factory) transformer = GeoJsonTransformer(data_factory) geojson = None for iter in range(len(police_beats_geojson)): police_beat = ComplexPolygonStrategyFactory().from_geojson(police_beats_geojson[iter]["geometry"]) # Generate the data set of interest set_id = str(uuid4()) age_limit = datetime.datetime.now() age_limit -= datetime.timedelta(hours=1) age_limit = age_limit.timetuple() bounding_box = police_beat.get_bounding_box() keywords = ["Mayday", "parade"] print "Bounding Box:", bounding_box print "Set ID:", set_id
class TestWorker: ''' Tests the Worker class. ''' def setup(self): ''' Set up before each test. ''' # Set up the testing configuration settings config = ConfigParser() config.add_section('worker') config.set('worker', 'batch_copy_size', '10') # Set up the mock testing components self.result_queue = MockResultQueue() self.task_queue = MockTaskQueue() self.data_factory = MockDataFactory() # Set up the Worker instance to be tested self.worker = Worker(config, self.result_queue, self.task_queue, self.data_factory) def test_calculate_trending_topics(self): ''' Tests the calculate_trending_topics function. ''' self.data_factory.data = [ MockData({'id' : '1', 'content' : "#yolo something about life!"}), MockData({'id' : '2', 'content' : "#yoLO! something else about life #YOLO but more exciting!"}), MockData({'id' : '3', 'content' : "#KendrickLamar Money trees, shake em!"}), MockData({'id' : '4', 'content' : "#SHESELLSSEASHELLS!!!! #Kendricklamar Snausages!"}) ] # Run the calculate trending topics trending_topics = self.worker.calculate_trending_topics('mock_data_set') # Check the results assert len(trending_topics) == 3, len(trending_topics) assert trending_topics[0][0] == '#YOLO', trending_topics[0][0] assert trending_topics[0][1] == 3, trending_topics[0][1] assert trending_topics[1][0] == '#KENDRICKLAMAR', trending_topics[1][0] assert trending_topics[1][1] == 2, trending_topics[1][1] assert trending_topics[2][0] == '#SHESELLSSEASHELLS', trending_topics[2][0] assert trending_topics[2][1] == 1, trending_topics[2][1] def test_filter_data(self): ''' Tests the _filter_data() function ''' # Set up the input data self.data_factory.data = [ MockData({'id' : '1', 'content' : "This is a test message that should be filtered out!"}), MockData({'id' : '2', 'content' : "There's a gun in our school!; Don't filter me!"}), MockData({'id' : '3', 'content' : "There's a gunman in our school!; Filter me!"}) ] # Run the filter function kwargs = { 'in_data_set_id' : 'in_data_set_id', 'out_data_set_id' : 'out_data_set_id', 'keywords' : ['GUN'] } self.worker._filter_data(**kwargs) # Check the results assert len(self.result_queue.posted_results) == 1, len(self.result_queue.posted_results) assert self.result_queue.posted_results[0]['set_id'] == 'out_data_set_id', \ self.result_queue.posted_results[0]['set_id'] assert len(self.data_factory.copied_data) == 1, len(self.data_factory.copied_data) assert self.data_factory.copied_data[0].get_datum_id() == '2', \ self.data_factory.copied_data[0].get_datum_id() assert self.data_factory.copied_data_set_id == 'out_data_set_id', \ self.data_factory.copied_data_set_id def test_strip_lingering_whitespace(self): ''' Tests using the _filter_data() function with content tokens burdened by hanging whitespace. ''' # Set up the input data self.data_factory.data = [ MockData({'id' : '1', 'content' : "'gun,"}), MockData({'id' : '2', 'content' : "gun;"}), MockData({'id' : '3', 'content' : "gun."}) ] # Run the filter function kwargs = { 'in_data_set_id' : 'in_data_set_id', 'out_data_set_id' : 'out_data_set_id', 'keywords' : ['GUN'] } self.worker._filter_data(**kwargs) # Check the results assert len(self.data_factory.copied_data) == 3, len(self.data_factory.copied_data)
# Load the config settings config = ConfigParser() configFile = open('config/qa_smcity.conf') config.readfp(configFile) configFile.close() # Load the GeoJSON description of Franklin county franklin_geojson = geojson.loads(open('manual_tests/franklin.geojson').read()) franklin_county = ComplexPolygonStrategyFactory().from_geojson(franklin_geojson['features'][0]['geometry']) # Set up the components data_factory = AwsDataFactory(config) result_queue = MockResultQueue() task_queue = MockTaskQueue() worker = Worker(config, result_queue, task_queue, data_factory) transformer = GeoJsonTransformer(data_factory) # Generate the data set of interest set_id = str(uuid4()) age_limit = datetime.datetime.now() age_limit += datetime.timedelta(hours=5) - datetime.timedelta(hours=1) age_limit = age_limit.timetuple() bounding_box = franklin_county.get_bounding_box() print "Bounding Box:", bounding_box print "Set ID:", set_id print "Time:", age_limit kwargs = { 'num_segments' : 4, 'in_data_set_id' : 'global', 'out_data_set_id' : set_id,
print "Loading the config settings..." config = ConfigParser() configFile = open('config/seattle_smcity.conf') config.readfp(configFile) configFile.close() # Load the GeoJSON description of the police beats police_beats_geojson = geojson.loads(open('seattle_slides/seattle_police_beats.geojson').read()) police_beats = ComplexPolygonStrategyFactory().from_geojson(police_beats_geojson['features'][0]['geometry']) # Set up the components data_factory = AwsDataFactory(config) result_queue = MockResultQueue() task_queue = MockTaskQueue() worker = Worker(config, result_queue, task_queue, data_factory) transformer = GeoJsonTransformer(data_factory) # Generate the data set of interest set_id = str(uuid4()) age_limit = datetime.datetime.now() age_limit -= datetime.timedelta(days=3) age_limit = age_limit.timetuple() bounding_box = police_beats.get_bounding_box() print "Bounding Box:", bounding_box print "Set ID:", set_id print "Time:", age_limit kwargs = { 'num_segments' : 4, 'in_data_set_id' : 'global', 'out_data_set_id' : set_id,