def setUp(self):
super(TestExporter, self).setUp()
config = DummyConfig({
'export:test': {
'url': None,
'skip_keys': 'export_source',
'batch': '3',
},
'export:everything': {
'url': '',
'batch': '5',
},
'export:no_test': {
'skip_keys': 'test_1 test\ntest:-1',
'batch': '2',
},
'export:invalid_ftp': {
'url': 'ftp://127.0.0.1:9/',
'batch': '5',
},
'export:invalid': {
'url': 'no_url',
'batch': '5',
},
})
self.celery_app.export_queues = queues = configure_export(
self.redis_client, config)
self.test_queue_key = queues['test'].queue_key()
self.session.add(ApiKey(valid_key='test2', log_submit=True))
self.session.flush()
def _init_searcher(self, klass):
return klass(
settings=DummyConfig({}),
geoip_db=self.geoip_db,
raven_client=self.raven_client,
redis_client=self.redis_client,
stats_client=self.stats_client,
)
def setUp(self):
super(TestInternal, self).setUp()
config = DummyConfig({
'export:internal': {
'url': 'internal://',
'batch': '0',
},
})
self.celery_app.app_config = config
def setUp(self):
super(TestS3, self).setUp()
config = DummyConfig({
'export:backup': {
'url': 's3://bucket/backups/{api_key}/{year}/{month}/{day}',
'batch': '3',
},
})
self.celery_app.app_config = config
def setUp(self):
super(TestGeosubmit, self).setUp()
config = DummyConfig({
'export:test': {
'url': 'http://127.0.0.1:9/v2/geosubmit?key=external',
'batch': '3',
},
})
self.celery_app.app_config = config
def setUp(self):
super(TestS3Uploader, self).setUp()
config = DummyConfig({
'export:backup': {
'url': 's3://bucket/backups/{api_key}/{year}/{month}/{day}',
'batch': '3',
},
})
self.celery_app.export_queues = configure_export(
self.redis_client, config)
def setUp(self):
super(TestGeosubmitUploader, self).setUp()
config = DummyConfig({
'export:test': {
'url': 'http://127.0.0.1:9/v2/geosubmit?key=external',
'batch': '3',
},
})
self.celery_app.export_queues = configure_export(
self.redis_client, config)
def setUp(self):
super(TestUploader, self).setUp()
config = DummyConfig({
'export:internal': {
'url': 'internal://',
'metadata': 'true',
'batch': '0',
},
})
self.celery_app.export_queues = configure_export(
self.redis_client, config)
def test_redis_config(self):
app_config = DummyConfig({
'ichnaea': {
'db_master': SQLURI,
'db_slave': SQLURI,
'redis_url': REDIS_URI,
}
})
app = _make_app(app_config=app_config,
_raven_client=self.raven_client,
_stats_client=self.stats_client)
self.assertTrue(app.app.registry.redis_client is not None)
def test_redis_config(self):
app_config = DummyConfig({
'cache': {
'cache_url': REDIS_URI,
},
})
app = _make_app(app_config=app_config,
_db_rw=self.db_rw,
_db_ro=self.db_ro,
_raven_client=self.raven_client,
_stats_client=self.stats_client)
redis_client = app.app.registry.redis_client
self.assertTrue(redis_client is not None)
self.assertEqual(redis_client.connection_pool.connection_kwargs['db'],
1)
def test_db_hooks(self):
app_config = DummyConfig(
{'ichnaea': {
'db_master': SQLURI,
'db_slave': SQLURI,
}})
app = _make_app(
app_config=app_config,
_raven_client=self.raven_client,
_redis_client=self.redis_client,
_stats_client=self.stats_client,
)
self.db_rw = app.app.registry.db_rw
self.db_ro = app.app.registry.db_ro
self.setup_session()
app.get('/stats_wifi.json', status=200)
def setUp(self):
super(TestExporter, self).setUp()
config = DummyConfig({
'export:test': {
'url': 'dummy://',
'skip_keys': 'export_source',
'batch': '3',
},
'export:everything': {
'url': 'dummy://',
'batch': '5',
},
'export:no_test': {
'url': 'dummy://',
'skip_keys': 'test_1 test\ntest:-1',
'batch': '2',
},
})
self.celery_app.app_config = config
ApiKeyFactory(valid_key='test2')
self.session.flush()
def test_db_config(self):
app_config = DummyConfig({
'database': {
'rw_url': SQLURI,
'ro_url': SQLURI,
},
})
app = _make_app(app_config=app_config,
_raven_client=self.raven_client,
_redis_client=self.redis_client,
_stats_client=self.stats_client,
)
db_rw = app.app.registry.db_rw
db_ro = app.app.registry.db_ro
# the configured databases are working
try:
self.assertTrue(db_rw.ping())
self.assertTrue(db_ro.ping())
finally:
# clean up the new db engine's _make_app created
db_rw.engine.pool.dispose()
db_ro.engine.pool.dispose()
def test_schedule(self):
app_config = DummyConfig({
'assets': {
'bucket': 'bucket',
},
'export:internal': {
'url': 'internal://',
'metadata': 'True',
'batch': '1000',
},
'export:backup': {
'url': 's3://bucket/directory/{api_key}/{year}/{month}/{day}',
'skip_keys': 'test',
'batch': '10000',
},
'export:outside': {
'url': 'https://localhost:9/some/api/url?key=export',
'skip_keys': 'test',
'batch': '10000',
},
'import:ocid': {
'url': 'https://localhost:9/downloads/',
'apikey': 'some_key',
},
})
tasks = set(schedule.celerybeat_schedule(app_config))
for name in ('cell-export-full', 'cell-export-diff'):
self.assertTrue(name in tasks)
for name in ('cell-import-external', 'monitor-ocid-import'):
self.assertTrue(name in tasks)
for name in ('gsm', 'wcdma', 'lte'):
self.assertTrue('update-cell-' + name in tasks)
for name in ('ne', 'nw', 'se', 'sw'):
self.assertTrue('update-datamap-' + name in tasks)
for i in range(16):
self.assertTrue('update-wifi-%x' % i in tasks)
# Some test-data constants
TEST_CONFIG = DummyConfig({
'assets': {
'bucket': 'localhost.bucket',
'url': 'http://127.0.0.1:7001/static/',
},
'export:test': {
'url': None,
'skip_keys': 'export',
'batch': '3',
},
'export:internal': {
'url': 'internal://',
'metadata': 'true',
'batch': '0',
},
'import:ocid': {
'url': 'http://127.0.0.1:9/downloads/',
'apikey': 'xxxxxxxx-yyyy-xxxx-yyyy-xxxxxxxxxxxx',
},
'locate:fallback': {
'url': 'http://127.0.0.1:9/?api',
'ratelimit': '10',
'ratelimit_expire': '60',
'ratelimit_interval': '5',
'cache_expire': '60',
},
})
GEOIP_DATA = {
SQL_BASE_STRUCTURE = os.path.join(DATA_DIRECTORY, 'base_structure.sql')
SQLURI = os.environ.get('SQLURI')
REDIS_URI = os.environ.get('REDIS_URI', 'redis://localhost:6379/1')
SESSION = {}
# Some test-data constants
TEST_CONFIG = DummyConfig({
'ichnaea': {
'assets_url': 'http://127.0.0.1:7001/static/',
's3_backup_bucket': 'localhost.bucket',
's3_assets_bucket': 'localhost.bucket',
'ocid_url': 'http://127.0.0.1:9/downloads/',
'ocid_apikey': 'xxxxxxxx-yyyy-xxxx-yyyy-xxxxxxxxxxxx',
},
'export:test': {
'url': None,
'source_apikey': 'export',
'batch': '3',
},
})
GEOIP_DATA = {
'London': {
'city': True,
'country_code': 'GB',
'country_name': 'United Kingdom',
'ip': '81.2.69.192',
'latitude': 51.5142,
'longitude': -0.0931,