def main(arguments):
import os.path
if len(arguments) == 0:
print('Usage: main.py <settings_file.ini> [truncate_data]')
print('Specify truncate_data if you wish to clear all the data')
print()
print('Example usage: main.py localhost.ini')
print('Example usage: main.py localhost.ini truncate_data')
exit(2)
settings_file = arguments[0]
if not os.path.isfile(settings_file):
print('Settings file not found: %s' % settings_file)
config_manager = ConfigManager.from_file(settings_file)
# for stat_collector in stat_collectors:
if len(arguments) == 2:
if arguments[1] == 'truncate_data':
print('not implemented')
# stat_collector.cleanup(config_manager)
stat_manager = StatManager(config_manager)
stat_manager.run()
def test_should_get_result_from_source(self):
manager = ConfigManager.from_file('test.ini')
MockSourceImpl.records.clear()
MockSourceImpl.records.append({'a': 10})
delta_queue = ClosableQueue()
sm = SourceManager.from_config_manager(config_manager, delta_queue)
sm.process_all_sources()
self.assertEquals(delta_queue.get().rows, [{'a': 10}])
def test_should_get_sources_from_multiple_servers(self):
cm = ConfigManager.from_file('multiple.ini')
source_manager = SourceManager.from_config_manager(cm, ClosableQueue())
self.assertEquals(len(source_manager.sources), 2)
def test_should_load_sql_server_source(self):
manager = ConfigManager.from_file('sql_server.ini')
delta_queue = ClosableQueue()
sm = SourceManager.from_config_manager(manager, delta_queue)
self.assertTrue(len(sm.sources) > 1)
import unittest
from config_manager import ConfigManager, SourceType
from query import Query
from source import Source
from source_manager import SourceManager
from mock_source import MockSourceImpl
from stoppable_worker import StoppableWorker, ClosableQueue
config_manager = ConfigManager.from_file('test.ini')
query = Query(key_column='a',
query_name='Mock',
mapping={},
non_data_fields=[],
get_data=lambda: ['Mocked result'])
class SourceManagerTests(unittest.TestCase):
def test_should_get_sources_from_type(self):
delta_queue = ClosableQueue()
sm = SourceManager.from_config_manager(config_manager, delta_queue)
mock = SourceType('MockSourceImpl', MockSourceImpl)
sources = sm.get_source_for_class(mock, config_manager)
self.assertEquals(len(sources), 1)
def test_should_add_source(self):
delta_queue = ClosableQueue()
sm = SourceManager.from_config_manager(config_manager, delta_queue)
def test_instantiate(self):
config_manager = ConfigManager.from_file('test.ini')
statman = StatManager(config_manager, SourceManager, ElasticMock)
# statman.run()
import unittest
from config_manager import ConfigManager
from source_manager import SourceManager
from sql_server_source import SQLServerSource, QueryStore
from stoppable_worker import ClosableQueue
config_manager = ConfigManager.from_file("sql_server.ini")
query_store = QueryStore()
class SQLServerSourceTestCase(unittest.TestCase):
def test_should_create_sql_server_source(self):
config = config_manager.get_config("localhost.master")
impl = SQLServerSource()
sources = impl.get_sources(config)
self.assertEquals(len(sources), len(query_store.queries))
f = sources[0]
self.assertIsNotNone(f.query)
self.assertIsNotNone(f.source_name)
self.assertIsNotNone(f.cache)
def test_should_execute_query(self):
delta_queue = ClosableQueue()
sm = SourceManager.from_config_manager(config_manager, delta_queue)
# for source in sm.sources:
# data = sm.get_data(source)
# self.assertTrue(len(data) > 0)
def test_should_get_multiple_sources_for_one_type(self):
cm = ConfigManager.from_file('multiple.ini')
configs = cm.get_config_for_source_name('MockSourceImpl')
self.assertEquals(len(configs), 2)
def main(arguments):
import os.path
if len(arguments) == 0:
print('Usage: main.py <settings_file.ini> [truncate_data]')
print('Specify truncate_data if you wish to clear all the data')
print()
print('Example usage: main.py localhost.ini')
print('Example usage: main.py localhost.ini truncate_data')
exit(2)
settings_file = arguments[0]
if not os.path.isfile(settings_file):
print('Settings file not found: %s' % settings_file)
config_manager = ConfigManager.from_file(settings_file)
es = ElasticsearchAPI.from_config_manager(config_manager)
# for stat_collector in stat_collectors:
if len(arguments) == 2:
if arguments[1] == 'truncate_data':
es.delete_index('police_events')
result = dumper.get(polisen_events)
rssEntries = dumper.parse_to_obj(result)
es.create_index('police_events')
mapping = {
"properties": {
"published": {
"type": "date",
"format": "date_hour_minute_second"
},
"title": {
"type": "string",
"index": "analyzed"
},
"link": {
"type": "string",
"index": "analyzed"
},
"summary": {
"type": "string",
"index": "analyzed"
},
"location": {
"type": "string",
"index": "not_analyzed"
},
"reported_date": {
"type": "string",
"index": "not_analyzed"
},
"report_type": {
"type": "string",
"index": "not_analyzed"
},
"location_street": {
"type": "string",
"index": "not_analyzed"
},
"location_commune": {
"type": "string",
"index": "not_analyzed"
},
"location_region": {
"type": "string",
"index": "not_analyzed"
},
"html_body": {
"type": "string",
"index": "analyzed"
},
}
}
es.set_mapping(index_name='police_events', doc_type='events', mapping=mapping)
existing_entries = es.find_ids([r['entry_id'] for r in rssEntries], index_name='police_events', doc_type='events')
new_entries = [e for e in rssEntries if e['entry_id'] not in existing_entries]
print('Found %i rss items of which %i are new' % (len(rssEntries), len(new_entries)))
# Getting the HTML body only for the new entries to reduce overhear
for entry in new_entries:
entry['html_body'] = dumper.get_link_body(entry['link'])
es.consume_all(new_entries, index_name='police_events', doc_type='events', id_column_name='entry_id')
