def wait_for_es(self):
if self.app.config.get('ELASTICSEARCH_URL'):
time.sleep(2)
helpers.full_user_import()
helpers.full_channel_import()
helpers.full_video_import()
time.sleep(2)
def import_to_es(prefix=None, **kwargs):
"""Import data into elasticsearch from the db"""
# NOTE: change this to be sensible
from rockpack.mainsite.core.es import helpers
i = helpers.DBImport()
if kwargs['terms_for_channel_only']:
i.import_video_channel_terms()
return
if kwargs['stars_only']:
i.import_video_stars()
return
if kwargs['restrictions_only']:
i.import_video_restrictions()
return
if kwargs['shares_only']:
i.import_channel_share()
return
if kwargs['suggestions_only']:
i.import_search_suggestions()
return
if not (kwargs['videos_only'] or kwargs['users_only']):
helpers.full_channel_import()
if not (kwargs['channels_only'] or kwargs['users_only']):
helpers.full_video_import(prefix=prefix)
if not (kwargs['channels_only'] or kwargs['videos_only']):
helpers.full_user_import()
def pytest_configure(config):
from rockpack.mainsite.core.es import mappings
mappings.CHANNEL_ALIAS = mappings.CHANNEL_INDEX = 'test_channel'
mappings.VIDEO_ALIAS = mappings.VIDEO_INDEX = 'test_video'
mappings.USER_ALIAS = mappings.USER_INDEX = 'test_user'
from rockpack.mainsite import app, init_app
app.config['TESTING'] = True
app.config['FORCE_INDEX_INSERT_REFRESH'] = True
app.config['DATABASE_URL'] = app.config.get('TEST_DATABASE_URL', 'sqlite://')
# import after setting DATABASE_URL
from rockpack.mainsite.core import dbapi
if app.config.get('ELASTICSEARCH_URL'):
from rockpack.mainsite.core.es import helpers
helpers.Indexing.create_all_indexes(rebuild=True)
helpers.Indexing.create_all_mappings()
if 'sqlite:' in app.config['DATABASE_URL']:
connection = dbapi.db.engine.raw_connection().connection
# Seems to be required for sub-transaction support:
connection.isolation_level = None
# Use group_concat instead of string_agg
from sqlalchemy import func
func.string_agg = func.group_concat
# For compatibility with postgres. XXX: can't return timedelta :-(
from datetime import datetime
connection.create_function('age', 1, lambda d: None)
# substitute postgres-specific "interval" expression
from rockpack.mainsite.services.user import api
from sqlalchemy import text
api.SUBSCRIPTION_VIDEO_FEED_THRESHOLD = text("datetime('now')")
api.ACTIVITY_LAST_ACTION_COMPARISON = "action = '%s'"
dbapi.sync_database(drop_all=True)
from wonder.common import timing
timing.log.level = 50
from test.test_helpers import install_mocks
from test.fixtures import install, all_data
install_mocks()
init_app()
# Explicityly load admin tables after app is loaded.
dbapi.sync_database(custom_modules=('rockpack.mainsite.admin', 'rockpack.mainsite.admin.auth', ))
install(*all_data)
if app.config.get('ELASTICSEARCH_URL'):
helpers.full_user_import()
helpers.full_channel_import()
helpers.full_video_import()
def update_indexes():
"""Updates all data in all indexes"""
job_control = JobControl.query.get('update_indexes')
start = job_control.last_run
stop = start + timedelta(seconds=60)
delay = datetime.utcnow() - stop
if delay < timedelta(0):
return
app.logger.info('update_indexes: from %s to %s', start, stop)
start_time = time.time()
helpers.full_user_import(start=start, stop=stop)
helpers.full_channel_import(start=start, stop=stop)
helpers.full_video_import(start=start, stop=stop)
app.logger.info('update_indexes: ran in %ds: delay %ds',
time.time() - start_time, delay.total_seconds())
record_timing('cron_processor.update_indexes.delay', delay.total_seconds())
job_control.last_run = stop
