def test_enabled_autodiscovery(app):
"""Test shared task detection."""
ext = InvenioCelery(app)
ext.load_entry_points()
assert 'conftest.shared_compute' in ext.celery.tasks.keys()
assert 'first_tasks.first_task' in ext.celery.tasks.keys()
assert 'second_tasks.second_task_a' in ext.celery.tasks.keys()
assert 'second_tasks.second_task_b' in ext.celery.tasks.keys()
def test_disabled_autodiscovery(app):
"""Test disabled discovery."""
ext = InvenioCelery(app, entry_point_group=None)
ext.load_entry_points()
assert 'conftest.shared_compute' in ext.celery.tasks.keys()
assert 'first_tasks.first_task' not in ext.celery.tasks.keys()
assert 'second_tasks.second_task_a' not in ext.celery.tasks.keys()
assert 'second_tasks.second_task_b' not in ext.celery.tasks.keys()
def test_only_first_tasks(app):
"""Test loading different entrypoint group."""
ext = InvenioCelery(app, entry_point_group='only_first_tasks')
ext.load_entry_points()
assert 'conftest.shared_compute' in ext.celery.tasks.keys()
assert 'first_tasks.first_task' in ext.celery.tasks.keys()
assert 'second_tasks.second_task_a' not in ext.celery.tasks.keys()
assert 'second_tasks.second_task_b' not in ext.celery.tasks.keys()
def test_disabled_autodiscovery(app):
"""Test disabled discovery."""
ext = InvenioCelery(app, entry_point_group=None)
ext.load_entry_points()
assert "conftest.shared_compute" in ext.celery.tasks.keys()
assert "first_tasks.first_task" not in ext.celery.tasks.keys()
assert "second_tasks.second_task_a" not in ext.celery.tasks.keys()
assert "second_tasks.second_task_b" not in ext.celery.tasks.keys()
def test_only_first_tasks(app):
"""Test loading different entrypoint group."""
ext = InvenioCelery(app, entry_point_group="only_first_tasks")
ext.load_entry_points()
assert "conftest.shared_compute" in ext.celery.tasks.keys()
assert "first_tasks.first_task" in ext.celery.tasks.keys()
assert "second_tasks.second_task_a" not in ext.celery.tasks.keys()
assert "second_tasks.second_task_b" not in ext.celery.tasks.keys()
def test_enabled_autodiscovery(app):
"""Test shared task detection."""
ext = InvenioCelery(app)
ext.load_entry_points()
assert "conftest.shared_compute" in ext.celery.tasks.keys()
assert "first_tasks.first_task" in ext.celery.tasks.keys()
assert "second_tasks.second_task_a" in ext.celery.tasks.keys()
assert "second_tasks.second_task_b" in ext.celery.tasks.keys()
def test_suspend_queues(app):
"""Test suspend queues."""
ext = InvenioCelery(app)
ext.celery.control = MagicMock()
c = {'i': 0}
def _mock():
c['i'] += 1
return [] if c['i'] > 1 else ['test']
ext.get_active_tasks = _mock
ext.suspend_queues(['feed'], sleep_time=0.1)
def test_suspend_queues(app):
"""Test suspend queues."""
ext = InvenioCelery(app)
ext.celery.control = MagicMock()
c = {"i": 0}
def _mock():
c["i"] += 1
return [] if c["i"] > 1 else ["test"]
ext.get_active_tasks = _mock
ext.suspend_queues(["feed"], sleep_time=0.1)
def test_suspend_queues(app):
"""Test suspend queues."""
ext = InvenioCelery(app)
ext.celery.control = MagicMock()
c = {'i': 0}
def _mock():
c['i'] += 1
return [] if c['i'] > 1 else ['test']
ext.get_active_tasks = _mock
ext.suspend_queues(['feed'], sleep_time=0.1)
def app(request):
"""Flask application fixture."""
# Set temporary instance path for sqlite
instance_path = tempfile.mkdtemp()
app = Flask('testapp', instance_path=instance_path)
app.config.update(
SQLALCHEMY_DATABASE_URI=os.environ.get('SQLALCHEMY_DATABASE_URI',
'sqlite:///test.db'),
INDEXER_REPLACE_REFS=True,
CELERY_ALWAYS_EAGER=True,
CELERY_RESULT_BACKEND="cache",
CELERY_CACHE_BACKEND="memory",
CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
JSONSCHEMAS_HOST='inveniosoftware.org',
OPENAIRE_OAI_LOCAL_SOURCE='invenio_openaire/data/oaire_local.sqlite',
TESTING=True,
)
app.url_map.converters['pid'] = PIDConverter
app.url_map.converters['pidpath'] = PIDPathConverter
LoginManager(app)
InvenioDB(app)
InvenioIndexer(app)
InvenioRecords(app)
InvenioCelery(app)
InvenioPIDStore(app)
InvenioOpenAIRE(app)
InvenioSearch(app)
InvenioJSONSchemas(app)
with app.app_context():
yield app
shutil.rmtree(instance_path)
def factory(**config):
app = Flask('testapp', instance_path=instance_path)
app.config.update(**config)
Babel(app)
InvenioDB(app)
InvenioCelery(app)
InvenioFlow(app)
return app
def _create_app(**config):
app_ = Flask(
__name__,
instance_path=instance_path,
)
app_.config.update(config)
InvenioCelery(app_)
InvenioDB(app_)
InvenioRecords(app_)
return app_
def test_init(app):
"""Test Celery initialization."""
celery = InvenioCelery(app)
assert app.extensions['invenio-celery'] == celery
called = {}
@celery.celery.task
def test1():
called['test1'] = True
test1.delay()
assert called['test1']
from invenio_jsonschemas import InvenioJSONSchemas
from invenio_pidstore import InvenioPIDStore
from invenio_records import InvenioRecords
from sqlalchemy_utils.functions import create_database, database_exists, \
drop_database
from invenio_openaire import InvenioOpenAIRE
from invenio_openaire.tasks import harvest_fundref, harvest_openaire_projects
# Create Flask application
app = Flask(__name__)
Babel(app)
FlaskCLI(app)
InvenioDB(app)
InvenioRecords(app)
InvenioCelery(app)
InvenioPIDStore(app)
InvenioOpenAIRE(app)
InvenioJSONSchemas(app)
app.config.update(
TESTING=True,
SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://postgres:postgres@localhost'
'/invenio_test',
CELERY_ALWAYS_EAGER=True,
BROKER_URL='redis://*****:*****@localhost:5672//',
# CELERY_ACCEPT_CONTENT=['json', 'msgpack', 'yaml'],
# CELERY_RESULT_SERIALIZER='msgpack',
# CELERY_TASK_SERIALIZER='msgpack',
)
def test_get_queues(app):
"""Test get queues."""
ext = InvenioCelery(app)
ext.celery.control = MagicMock()
assert ext.get_queues() == []
def app(mapping, schema):
instance_path = tempfile.mkdtemp()
app = Flask('testapp', instance_path=instance_path)
app.config.update(
JSONSCHEMAS_HOST="example.com",
SQLALCHEMY_TRACK_MODIFICATIONS=True,
SERVER_NAME='127.0.0.1:5000',
INVENIO_INSTANCE_PATH=instance_path,
DEBUG=True,
# in tests, api is not on /api but directly in the root
PIDSTORE_RECID_FIELD='pid',
FLASK_TAXONOMIES_URL_PREFIX='/2.0/taxonomies/',
RECORDS_REST_ENDPOINTS=RECORDS_REST_ENDPOINTS,
CELERY_BROKER_URL='amqp://*****:*****@localhost:5672//',
CELERY_TASK_ALWAYS_EAGER=True,
CELERY_RESULT_BACKEND='cache',
CELERY_CACHE_BACKEND='memory',
CELERY_TASK_EAGER_PROPAGATES=True)
app.secret_key = 'changeme'
print(os.environ.get("INVENIO_INSTANCE_PATH"))
InvenioDB(app)
OarepoTaxonomies(app)
OARepoReferences(app)
InvenioAccounts(app)
InvenioAccess(app)
Principal(app)
InvenioJSONSchemas(app)
InvenioSearch(app)
OARepoMappingIncludesExt(app)
InvenioRecords(app)
InvenioRecordsREST(app)
InvenioCelery(app)
OARepoValidate(app)
# Celery
print(app.config["CELERY_BROKER_URL"])
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = 'login'
@login_manager.user_loader
def basic_user_loader(user_id):
user_obj = User.query.get(int(user_id))
return user_obj
# app.register_blueprint(create_blueprint_from_app(app))
@app.route('/test/login/<int:id>', methods=['GET', 'POST'])
def test_login(id):
print("test: logging user with id", id)
response = make_response()
user = User.query.get(id)
login_user(user)
set_identity(user)
return response
app.extensions['invenio-search'].mappings["test"] = mapping
app.extensions["invenio-jsonschemas"].schemas["test"] = schema
app_loaded.send(app, app=app)
with app.app_context():
# app.register_blueprint(taxonomies_blueprint)
yield app
shutil.rmtree(instance_path)
def test_get_active_tasks(app):
"""Test get active tasks."""
ext = InvenioCelery(app)
ext.celery.control = MagicMock()
assert ext.get_active_tasks() == []
def test_disable_queue(app):
"""Test enable queues."""
ext = InvenioCelery(app)
ext.celery.control = MagicMock()
ext.disable_queue('feed')
SQLALCHEMY_TRACK_MODIFICATIONS=True,
))
Babel(app)
Menu(app)
InvenioDB(app)
InvenioREST(app)
InvenioAdmin(app)
InvenioAccounts(app)
InvenioAccess(app)
InvenioFilesREST(app)
app.register_blueprint(accounts_blueprint)
app.register_blueprint(blueprint)
celery = InvenioCelery(app).celery
@app.cli.group()
def fixtures():
"""Command for working with test data."""
@fixtures.command()
def files():
"""Load files."""
srcroot = dirname(dirname(__file__))
d = current_app.config['DATADIR']
if exists(d):
shutil.rmtree(d)
makedirs(d)
def test_bare_module_warning(app):
"""Test loading different entrypoint group."""
ext = InvenioCelery(app, entry_point_group='bare_module_tasks')
with pytest.warns(RuntimeWarning):
ext.load_entry_points()
def test_disable_queue(app):
"""Test enable queues."""
ext = InvenioCelery(app)
ext.celery.control = MagicMock()
ext.disable_queue('feed')
def test_get_active_tasks(app):
"""Test get active tasks."""
ext = InvenioCelery(app)
ext.celery.control = MagicMock()
assert ext.get_active_tasks() == []
def test_worker_loading(app):
"""Test that tasks are only loaded on the worker."""
ext = InvenioCelery(app)
assert 'first_tasks.first_task' not in ext.celery.tasks.keys()
ext.celery.loader.import_default_modules()
assert 'first_tasks.first_task' in ext.celery.tasks.keys()
def app():
instance_path = tempfile.mkdtemp()
app = Flask('testapp', instance_path=instance_path)
app.config.update(
JSONSCHEMAS_HOST="nusl.cz",
SQLALCHEMY_TRACK_MODIFICATIONS=True,
SERVER_NAME='127.0.0.1:5000',
INVENIO_INSTANCE_PATH=instance_path,
DEBUG=True,
# in tests, api is not on /api but directly in the root
PIDSTORE_RECID_FIELD='pid',
FLASK_TAXONOMIES_URL_PREFIX='/2.0/taxonomies/',
# RECORDS_REST_ENDPOINTS=RECORDS_REST_ENDPOINTS,
CELERY_BROKER_URL='amqp://*****:*****@localhost:5672//',
CELERY_TASK_ALWAYS_EAGER=True,
CELERY_RESULT_BACKEND='cache',
CELERY_CACHE_BACKEND='memory',
CELERY_TASK_EAGER_PROPAGATES=True,
SUPPORTED_LANGUAGES=["cs", "en"],
OAREPO_COMMUNITIES_ENDPOINTS=[],
ELASTICSEARCH_DEFAULT_LANGUAGE_TEMPLATE={
"type": "text",
"fields": {
"keywords": {
"type": "keyword"
}
}
})
app.secret_key = 'changeme'
print(os.environ.get("INVENIO_INSTANCE_PATH"))
InvenioDB(app)
OarepoTaxonomies(app)
OARepoReferences(app)
InvenioAccounts(app)
InvenioAccess(app)
Principal(app)
InvenioJSONSchemas(app)
InvenioSearch(app)
InvenioIndexer(app)
OARepoMappingIncludesExt(app)
InvenioRecords(app)
InvenioRecordsREST(app)
InvenioCelery(app)
NRNresults(app)
InvenioPIDStore(app)
OARepoValidate(app)
OARepoCommunities(app)
# Invenio Records Draft initialization
RecordsDraft(app)
app.url_map.converters['pid'] = PIDConverter
app.url_map.converters['commpid'] = CommunityPIDConverter
# Celery
print(app.config["CELERY_BROKER_URL"])
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = 'login'
@login_manager.user_loader
def basic_user_loader(user_id):
user_obj = User.query.get(int(user_id))
return user_obj
app.register_blueprint(create_blueprint_from_app(app))
@app.route('/test/login/<int:id>', methods=['GET', 'POST'])
def test_login(id):
print("test: logging user with id", id)
response = make_response()
user = User.query.get(id)
login_user(user)
set_identity(user)
return response
# app.extensions['invenio-search'].mappings["test"] = mapping
# app.extensions["invenio-jsonschemas"].schemas["test"] = schema
app_loaded.send(app, app=app)
with app.app_context():
# app.register_blueprint(taxonomies_blueprint)
print(app.url_map)
yield app
shutil.rmtree(instance_path)
def test_get_queues(app):
"""Test get queues."""
ext = InvenioCelery(app)
ext.celery.control = MagicMock()
assert ext.get_queues() == []
