def main(arguments): import os.path if len(arguments) == 0: print('Usage: main.py <settings_file.ini> [truncate_data]') print('Specify truncate_data if you wish to clear all the data') print() print('Example usage: main.py localhost.ini') print('Example usage: main.py localhost.ini truncate_data') exit(2) settings_file = arguments[0] if not os.path.isfile(settings_file): print('Settings file not found: %s' % settings_file) config_manager = ConfigManager.from_file(settings_file) # for stat_collector in stat_collectors: if len(arguments) == 2: if arguments[1] == 'truncate_data': print('not implemented') # stat_collector.cleanup(config_manager) stat_manager = StatManager(config_manager) stat_manager.run()
def test_should_get_result_from_source(self): manager = ConfigManager.from_file('test.ini') MockSourceImpl.records.clear() MockSourceImpl.records.append({'a': 10}) delta_queue = ClosableQueue() sm = SourceManager.from_config_manager(config_manager, delta_queue) sm.process_all_sources() self.assertEquals(delta_queue.get().rows, [{'a': 10}])
def test_should_get_sources_from_multiple_servers(self): cm = ConfigManager.from_file('multiple.ini') source_manager = SourceManager.from_config_manager(cm, ClosableQueue()) self.assertEquals(len(source_manager.sources), 2)
def test_should_load_sql_server_source(self): manager = ConfigManager.from_file('sql_server.ini') delta_queue = ClosableQueue() sm = SourceManager.from_config_manager(manager, delta_queue) self.assertTrue(len(sm.sources) > 1)
import unittest from config_manager import ConfigManager, SourceType from query import Query from source import Source from source_manager import SourceManager from mock_source import MockSourceImpl from stoppable_worker import StoppableWorker, ClosableQueue config_manager = ConfigManager.from_file('test.ini') query = Query(key_column='a', query_name='Mock', mapping={}, non_data_fields=[], get_data=lambda: ['Mocked result']) class SourceManagerTests(unittest.TestCase): def test_should_get_sources_from_type(self): delta_queue = ClosableQueue() sm = SourceManager.from_config_manager(config_manager, delta_queue) mock = SourceType('MockSourceImpl', MockSourceImpl) sources = sm.get_source_for_class(mock, config_manager) self.assertEquals(len(sources), 1) def test_should_add_source(self): delta_queue = ClosableQueue() sm = SourceManager.from_config_manager(config_manager, delta_queue)
def test_instantiate(self): config_manager = ConfigManager.from_file('test.ini') statman = StatManager(config_manager, SourceManager, ElasticMock) # statman.run()
import unittest from config_manager import ConfigManager from source_manager import SourceManager from sql_server_source import SQLServerSource, QueryStore from stoppable_worker import ClosableQueue config_manager = ConfigManager.from_file("sql_server.ini") query_store = QueryStore() class SQLServerSourceTestCase(unittest.TestCase): def test_should_create_sql_server_source(self): config = config_manager.get_config("localhost.master") impl = SQLServerSource() sources = impl.get_sources(config) self.assertEquals(len(sources), len(query_store.queries)) f = sources[0] self.assertIsNotNone(f.query) self.assertIsNotNone(f.source_name) self.assertIsNotNone(f.cache) def test_should_execute_query(self): delta_queue = ClosableQueue() sm = SourceManager.from_config_manager(config_manager, delta_queue) # for source in sm.sources: # data = sm.get_data(source) # self.assertTrue(len(data) > 0)
def test_should_get_multiple_sources_for_one_type(self): cm = ConfigManager.from_file('multiple.ini') configs = cm.get_config_for_source_name('MockSourceImpl') self.assertEquals(len(configs), 2)
def main(arguments): import os.path if len(arguments) == 0: print('Usage: main.py <settings_file.ini> [truncate_data]') print('Specify truncate_data if you wish to clear all the data') print() print('Example usage: main.py localhost.ini') print('Example usage: main.py localhost.ini truncate_data') exit(2) settings_file = arguments[0] if not os.path.isfile(settings_file): print('Settings file not found: %s' % settings_file) config_manager = ConfigManager.from_file(settings_file) es = ElasticsearchAPI.from_config_manager(config_manager) # for stat_collector in stat_collectors: if len(arguments) == 2: if arguments[1] == 'truncate_data': es.delete_index('police_events') result = dumper.get(polisen_events) rssEntries = dumper.parse_to_obj(result) es.create_index('police_events') mapping = { "properties": { "published": { "type": "date", "format": "date_hour_minute_second" }, "title": { "type": "string", "index": "analyzed" }, "link": { "type": "string", "index": "analyzed" }, "summary": { "type": "string", "index": "analyzed" }, "location": { "type": "string", "index": "not_analyzed" }, "reported_date": { "type": "string", "index": "not_analyzed" }, "report_type": { "type": "string", "index": "not_analyzed" }, "location_street": { "type": "string", "index": "not_analyzed" }, "location_commune": { "type": "string", "index": "not_analyzed" }, "location_region": { "type": "string", "index": "not_analyzed" }, "html_body": { "type": "string", "index": "analyzed" }, } } es.set_mapping(index_name='police_events', doc_type='events', mapping=mapping) existing_entries = es.find_ids([r['entry_id'] for r in rssEntries], index_name='police_events', doc_type='events') new_entries = [e for e in rssEntries if e['entry_id'] not in existing_entries] print('Found %i rss items of which %i are new' % (len(rssEntries), len(new_entries))) # Getting the HTML body only for the new entries to reduce overhear for entry in new_entries: entry['html_body'] = dumper.get_link_body(entry['link']) es.consume_all(new_entries, index_name='police_events', doc_type='events', id_column_name='entry_id')