def test_celery():
"""Test celery."""
from invenio_logging.sentry import InvenioLoggingSentry
app = Flask('testapp')
app.config.update(dict(
SENTRY_DSN='http://*****:*****@localhost/0',
SENTRY_TRANSPORT='raven.transport.http.HTTPTransport',
LOGGING_SENTRY_CELERY=True,
CELERY_ALWAYS_EAGER=True,
CELERY_RESULT_BACKEND="cache",
CELERY_CACHE_BACKEND="memory",
CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
))
InvenioLoggingSentry(app)
celery = create_celery_app(app)
@celery.task
def test_task():
logger = get_task_logger(__name__)
logger.error('CELERYTEST')
httpretty.register_uri(
httpretty.POST,
'http://localhost/api/0/store/',
body='[{"title": "Test"}]',
content_type='application/json',
status=200,
)
test_task()
# Give sentry time to send request
from time import sleep
sleep(1)
assert httpretty.last_request().path == '/api/0/store/'
def app(env_config, default_config):
"""Flask application fixture."""
app = create_app(**default_config)
# FIXME: Needs fixing flask_celeryext,
# which once creates the first celery app, the flask_app that is set
# is never released from the global state, even if you create a new
# celery application. We need to unset the "flask_app" manually.
from celery import current_app as cca
cca = cca._get_current_object()
delattr(cca, "flask_app")
celery_app = create_celery_app(app)
# FIXME: When https://github.com/inveniosoftware/flask-celeryext/issues/35
# is closed and Flask-CeleryExt is released, this can be removed.
class _TestAppContextTask(Task):
abstract = True
def __call__(self, *args, **kwargs):
if flask_current_app:
return Task.__call__(self, *args, **kwargs)
with self.app.flask_app.app_context():
return Task.__call__(self, *args, **kwargs)
celery_app.Task = _TestAppContextTask
celery_app.set_current()
with app.app_context():
yield app
def test_factory():
"""Test of factory method."""
# Set the current Celery application
c = Celery('mycurrent')
c.set_current()
app = Flask("myapp")
celery = create_celery_app(app)
assert celery
assert celery.flask_app == app
def test_appctx_task():
"""Test execution of Celery task with application context."""
app = Flask("myapp")
app.config.from_object(eager_conf)
celery = create_celery_app(app)
@celery.task
def appctx():
return current_app.name
r = appctx.delay()
assert r.result == "myapp"
def app(env_config, default_config):
"""Flask application fixture."""
app = create_app(**default_config)
# FIXME: Needs fixing flask_celeryext,
# which once creates the first celery app, the flask_app that is set
# is never released from the global state, even if you create a new
# celery application. We need to unset the "flask_app" manually.
from celery import current_app as cca
cca = cca._get_current_object()
delattr(cca, "flask_app")
celery_app = create_celery_app(app)
celery_app.set_current()
with app.app_context():
yield app
def app(env_config, default_config):
"""Flask application fixture."""
app = create_app(**default_config)
# FIXME: Needs fixing flask_celeryext,
# which once creates the first celery app, the flask_app that is set
# is never released from the global state, even if you create a new
# celery application. We need to unset the "flask_app" manually.
from celery import current_app as cca
cca = cca._get_current_object()
delattr(cca, "flask_app")
celery_app = create_celery_app(app)
celery_app.set_current()
with app.app_context():
yield app
def test_task_logger_propagation():
"""Test log propagation of Celery task."""
app = Flask("myapp")
app.config.from_object(eager_conf)
celery = create_celery_app(app)
celery.log.setup_task_loggers()
@celery.task()
def logtask():
logger = get_task_logger(__name__)
while getattr(logger, 'parent', None):
assert logger.propagate == 1
logger = logger.parent
logtask.delay()
def celeryfactory(app):
"""Create a Celery application based on Flask application.
.. versionadded:: v0.2.0
:param app: Flask application instance.
"""
try:
# Check if celery application has already been created.
celery = app.extensions['flask-celeryext'].celery
except KeyError:
celery = create_celery_app(app)
load_tasks(app)
return celery
def test_appctx_task():
"""Test execution of Celery task with application context."""
app = Flask("myapp")
app.config.from_object(eager_conf)
# Set the current Celery application
c = Celery('mycurrent')
c.set_current()
celery = create_celery_app(app)
@celery.task
def appctx():
return current_app.name
r = appctx.delay()
assert r.result == "myapp"
def test_reqctx_task():
"""Test execution of Celery task with request context."""
app = Flask("myapp")
app.config.from_object(eager_conf)
celery = create_celery_app(app)
@celery.task(base=RequestContextTask)
def reqctx():
return request.method
@celery.task(base=AppContextTask)
def reqctx2():
return request.method
r = reqctx.delay()
assert r.result == "GET"
assert pytest.raises(RuntimeError, reqctx2.delay)
def test_reqctx_task():
"""Test execution of Celery task with request context."""
app = Flask("myapp")
app.config.from_object(eager_conf)
celery = create_celery_app(app)
@celery.task(base=RequestContextTask)
def reqctx():
return request.method
@celery.task(base=AppContextTask)
def reqctx2():
return request.method
r = reqctx.delay()
assert r.result == "GET"
assert pytest.raises(RuntimeError, reqctx2.delay)
def test_subtask_and_eager_dont_create_new_app_context(mocker):
app = Flask("myapp")
app.config.from_object(eager_conf)
# Set the current Celery application
c = Celery('mycurrent')
c.set_current()
celery = create_celery_app(app)
spy = mocker.spy(app, 'app_context')
@celery.task
def subtask():
return current_app.name
@celery.task
def maintask():
future = subtask.delay()
return future.result
r = maintask.delay()
assert r.result == "myapp"
assert spy.call_count == 1
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2017-2018 CERN.
#
# Invenio is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
"""Celery application for Invenio flavours."""
from __future__ import absolute_import, print_function
from flask_celeryext import create_celery_app
from .factory import create_ui
celery = create_celery_app(create_ui(
SENTRY_TRANSPORT='raven.transport.http.HTTPTransport',
RATELIMIT_ENABLED=False,
))
"""Celery application for Invenio.
Overrides SENTRY_TRANSPORT wih synchronous HTTP transport since Celery does not
deal nicely with the default threaded transport.
"""
# Trigger an app log message upon import. This makes Sentry logging
# work with `get_task_logger(__name__)`.
celery.flask_app.logger.info('Created Celery app')
<rdf:RDF xmlns="http://www.w3.org/2004/02/skos/core#"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#">
<Concept rdf:about="http://cern.ch/thesauri/HEPontology.rdf#Higgsparticle">
<prefLabel xml:lang="en">Higgs particle</prefLabel>
<altLabel xml:lang="en">Higgs boson</altLabel>
<hiddenLabel xml:lang="en">Higgses</hiddenLabel>
<note xml:lang="en">core</note>
</Concept>
</rdf:RDF>
'''
HIGGS_ONTOLOGY_FILE = tempfile.NamedTemporaryFile()
HIGGS_ONTOLOGY_FILE.write(HIGGS_ONTOLOGY)
HIGGS_ONTOLOGY_FILE.flush()
print('Using ontology file %s' % HIGGS_ONTOLOGY_FILE.name)
celery = create_celery_app(
create_app(
LOGGING_SENTRY_CELERY=True,
HEP_ONTOLOGY_FILE=HIGGS_ONTOLOGY_FILE.name,
)
)
# We don't want to log to Sentry backoff errors
logging.getLogger('backoff').propagate = 0
# INSPIRE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# INSPIRE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with INSPIRE. If not, see <http://www.gnu.org/licenses/>.
#
# In applying this license, CERN does not waive the privileges and immunities
# granted to it by virtue of its status as an Intergovernmental Organization
# or submit itself to any jurisdiction.
"""INSPIREHEP Celery app instantiation."""
from __future__ import absolute_import, division, print_function
import logging
from flask_celeryext import create_celery_app
from .factory import create_app
celery = create_celery_app(create_app(LOGGING_SENTRY_CELERY=True))
# We don't want to log to Sentry backoff errors
logging.getLogger('backoff').propagate = 0
def test_cli():
"""Test CLI."""
app = Flask(__name__)
app.config.update(
CELERY_ALWAYS_EAGER=True,
CELERY_CACHE_BACKEND="memory",
CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
CELERY_RESULT_BACKEND="cache",
SECRET_KEY="CHANGE_ME",
SECURITY_PASSWORD_SALT="CHANGE_ME_ALSO",
SQLALCHEMY_DATABASE_URI=os.environ.get(
'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db'
),
)
FlaskCLI(app)
InvenioDB(app)
InvenioRecords(app)
create_celery_app(app)
runner = CliRunner()
script_info = ScriptInfo(create_app=lambda info: app)
assert len(db.metadata.tables) == 3
from invenio_records.models import RecordMetadata as RM
# Test merging a base another file.
with runner.isolated_filesystem():
with open('record.json', 'wb') as f:
f.write(json.dumps(
{"title": "Test"}, ensure_ascii=False
).encode('utf-8'))
with open('record.patch', 'wb') as f:
f.write(json.dumps([{
"op": "replace", "path": "/title", "value": "Patched Test"
}], ensure_ascii=False).encode('utf-8'))
runner.invoke(db_cmd, ['init'], obj=script_info)
runner.invoke(db_cmd, ['create'], obj=script_info)
result = runner.invoke(cli.records, [], obj=script_info)
assert result.exit_code == 0
with app.app_context():
assert RM.query.count() == 0
result = runner.invoke(cli.records, ['create', 'record.json'],
obj=script_info)
assert result.exit_code == 0
with app.app_context():
assert RM.query.count() == 1
recid = RM.query.first().id
result = runner.invoke(cli.records,
['patch', 'record.patch', '-r', recid],
obj=script_info)
assert result.exit_code == 0
with app.app_context():
record = Record.get_record(recid)
assert record['title'] == 'Patched Test'
assert record.model.version_id == 2
runner.invoke(db_cmd, ['drop', '--yes-i-know'], obj=script_info)
runner.invoke(db_cmd, ['destroy', '--yes-i-know'], obj=script_info)
from flask_celeryext import create_celery_app # from flask_appfactory.celery import celeryfactory from ercc import create_app # celery = celeryfactory(create_app()) celery = create_celery_app(create_app())
from invenio_records import InvenioRecords
# Create Flask application
app = Flask(__name__)
app.config.update(
CELERY_ALWAYS_EAGER=True,
CELERY_CACHE_BACKEND="memory",
CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
CELERY_RESULT_BACKEND="cache",
SECRET_KEY="CHANGE_ME",
SECURITY_PASSWORD_SALT="CHANGE_ME_ALSO",
)
db_uri = os.environ.get('SQLALCHEMY_DATABASE_URI')
if db_uri is not None:
app.config['SQLALCHEMY_DATABASE_URI'] = db_uri
FlaskCLI(app)
InvenioDB(app)
InvenioRecords(app)
celery = create_celery_app(app)
@app.route("/<uuid>")
def index(uuid):
"""Retrieve record."""
from invenio_records.api import Record
return jsonify(Record.get_record(uuid))
#
# You should have received a copy of the GNU General Public License
# along with Zenodo; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.
"""Zenodo celery application object."""
from __future__ import absolute_import, print_function
from flask_celeryext import create_celery_app
from .factory import create_celery
celery = create_celery_app(create_celery(
SENTRY_TRANSPORT='raven.transport.http.HTTPTransport'
))
"""Celery application for Zenodo.
Override SENTRY_TRANSPORT since Celery does not deal nicely with the default
threaded transport.
"""
# Trigger an app log message upon import. Somehow this makes Sentry logging
# work with `get_task_logger(__name__)`.
celery.flask_app.logger.info('Created Celery app')
def test_factory():
"""Test of factory method."""
app = Flask("myapp")
celery = create_celery_app(app)
assert celery
assert celery.flask_app == app
#
# INSPIRE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with INSPIRE. If not, see <http://www.gnu.org/licenses/>.
#
# In applying this license, CERN does not waive the privileges and immunities
# granted to it by virtue of its status as an Intergovernmental Organization
# or submit itself to any jurisdiction.
"""INSPIREHEP Celery app instantiation."""
from __future__ import absolute_import, division, print_function
import logging
from flask_celeryext import create_celery_app
from .factory import create_app
celery = create_celery_app(
create_app(LOGGING_SENTRY_CELERY=True)
)
# We don't want to log to Sentry backoff errors
logging.getLogger('backoff').propagate = 0
HIGGS_ONTOLOGY = '''<?xml version="1.0" encoding="UTF-8" ?>
<rdf:RDF xmlns="http://www.w3.org/2004/02/skos/core#"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#">
<Concept rdf:about="http://cern.ch/thesauri/HEPontology.rdf#Higgsparticle">
<prefLabel xml:lang="en">Higgs particle</prefLabel>
<altLabel xml:lang="en">Higgs boson</altLabel>
<hiddenLabel xml:lang="en">Higgses</hiddenLabel>
<note xml:lang="en">core</note>
</Concept>
</rdf:RDF>
'''
HIGGS_ONTOLOGY_FILE = tempfile.NamedTemporaryFile()
HIGGS_ONTOLOGY_FILE.write(HIGGS_ONTOLOGY)
HIGGS_ONTOLOGY_FILE.flush()
print('Using ontology file %s' % HIGGS_ONTOLOGY_FILE.name)
celery = create_celery_app(
create_app(
LOGGING_SENTRY_CELERY=True,
HEP_ONTOLOGY_FILE=HIGGS_ONTOLOGY_FILE.name,
))
# We don't want to log to Sentry backoff errors
logging.getLogger('backoff').propagate = 0
# Copyright (C) 2015 CERN. # # HEPData is free software; you can redistribute it # and/or modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of the # License, or (at your option) any later version. # # HEPData is distributed in the hope that it will be # useful, but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with HEPData; if not, write to the # Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, # MA 02111-1307, USA. # # In applying this license, CERN does not # waive the privileges and immunities granted to it by virtue of its status # as an Intergovernmental Organization or submit itself to any jurisdiction. """HEPData celery application object.""" from __future__ import absolute_import, print_function from flask_celeryext import create_celery_app from .factory import create_app celery = create_celery_app(create_app())
# Copyright (C) 2016 CERN.
#
# CERN Analysis Preservation Framework is free software; you can redistribute
# it and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# CERN Analysis Preservation Framework is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with CERN Analysis Preservation Framework; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.
"""CAP Celery Invenio configuration."""
from __future__ import absolute_import, print_function
from flask_celeryext import create_celery_app
from cap.factory import create_api
celery = create_celery_app(
create_api(SENTRY_TRANSPORT='raven.transport.http.HTTPTransport'))
FlaskCLI(app)
InvenioDB(app)
InvenioRecords(app)
InvenioCelery(app)
InvenioPIDStore(app)
InvenioOpenAIRE(app)
InvenioJSONSchemas(app)
app.config.update(
TESTING=True,
SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://postgres:postgres@localhost'
'/invenio_test',
CELERY_ALWAYS_EAGER=True,
BROKER_URL='redis://*****:*****@localhost:5672//',
# CELERY_ACCEPT_CONTENT=['json', 'msgpack', 'yaml'],
# CELERY_RESULT_SERIALIZER='msgpack',
# CELERY_TASK_SERIALIZER='msgpack',
)
# celery_ext = InvenioCelery(app)
celery = create_celery_app(app)
if __name__ == "__main__":
with app.app_context():
if database_exists(str(db.engine.url)):
drop_database(str(db.engine.url))
create_database(str(db.engine.url))
db.create_all()
harvest_fundref()
harvest_openaire_projects()
def do_finalise(recid, publication_record=None, force_finalise=False,
commit_message=None, send_tweet=False, update=False, convert=True,
send_email=True):
"""
Creates record SIP for each data record with a link to the associated
publication.
:param synchronous: if true then workflow execution and creation is
waited on, then everything is indexed in one go.
If False, object creation is asynchronous, however reindexing is not
performed. This is only really useful for the full migration of
content.
"""
print('Finalising record {}'.format(recid))
hep_submission = get_latest_hepsubmission(publication_recid=recid)
generated_record_ids = []
if hep_submission \
and (force_finalise or hep_submission.coordinator == int(current_user.get_id())):
submissions = DataSubmission.query.filter_by(
publication_recid=recid,
version=hep_submission.version).all()
version = hep_submission.version
existing_submissions = {}
if hep_submission.version > 1 or update:
# we need to determine which are the existing record ids.
existing_data_records = get_records_matching_field(
'related_publication', recid, doc_type=CFG_DATA_TYPE)
for record in existing_data_records["hits"]["hits"]:
if "recid" in record["_source"]:
existing_submissions[record["_source"]["title"]] = \
record["_source"]["recid"]
delete_item_from_index(record["_id"],
doc_type=CFG_DATA_TYPE, parent=record["_source"]["related_publication"])
current_time = "{:%Y-%m-%d %H:%M:%S}".format(datetime.utcnow())
for submission in submissions:
finalise_datasubmission(current_time, existing_submissions,
generated_record_ids,
publication_record, recid, submission,
version)
try:
record = get_record_by_id(recid)
# If we have a commit message, then we have a record update.
# We will store the commit message and also update the
# last_updated flag for the record.
record['hepdata_doi'] = hep_submission.doi
# The last updated date will be the current date (if record not migrated from the old site).
if hep_submission.coordinator > 1:
hep_submission.last_updated = datetime.utcnow()
if commit_message:
commit_record = RecordVersionCommitMessage(
recid=recid,
version=version,
message=str(commit_message))
db.session.add(commit_record)
record['last_updated'] = datetime.strftime(
hep_submission.last_updated, '%Y-%m-%d %H:%M:%S')
record['version'] = version
record.commit()
hep_submission.inspire_id = record['inspire_id']
hep_submission.overall_status = "finished"
db.session.add(hep_submission)
db.session.commit()
create_celery_app(current_app)
# only mint DOIs if not testing.
if not current_app.config.get('TESTING', False):
generate_dois_for_submission.delay(inspire_id=hep_submission.inspire_id, version=version)
log.info("Generated DOIs for ins{0}".format(hep_submission.inspire_id))
# Reindex everything.
index_record_ids([recid] + generated_record_ids)
push_data_keywords(pub_ids=[recid])
try:
admin_indexer = AdminIndexer()
admin_indexer.index_submission(hep_submission)
except ConnectionTimeout as ct:
log.error('Unable to add ins{0} to admin index.\n{1}'.format(hep_submission.inspire_id, ct))
if send_email:
send_finalised_email(hep_submission)
if convert:
for file_format in ['yaml', 'csv', 'yoda', 'root']:
convert_and_store.delay(hep_submission.inspire_id, file_format, force=True)
if send_tweet:
site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')
tweet(record.get('title'), record.get('collaborations'),
site_url + '/record/ins{0}'.format(record.get('inspire_id')), version)
return json.dumps({"success": True, "recid": recid,
"data_count": len(submissions),
"generated_records": generated_record_ids})
except NoResultFound:
print('No record found to update. Which is super strange.')
else:
return json.dumps(
{"success": False, "recid": recid,
"errors": ["You do not have permission to finalise this "
"submission. Only coordinators can do that."]})
