def setUp(self):
unittest.TestCase.setUp(self)
self.proj_home = tasks.app.conf['PROJ_HOME']
self.grobid_service = tasks.app.conf['GROBID_SERVICE']
self._app = tasks.app
self.app = app.ADSFulltextCelery('test', proj_home=self.proj_home, local_config=\
{
})
tasks.app = self.app # monkey-patch the app object
def setUp(self):
unittest.TestCase.setUp(self)
self.proj_home = tasks.app.conf['PROJ_HOME']
self.grobid_service = tasks.app.conf['GROBID_SERVICE']
self._app = tasks.app
self.app = app.ADSFulltextCelery('test', proj_home=self.proj_home, local_config=\
{
"CELERY_ALWAYS_EAGER": False,
"CELERY_EAGER_PROPAGATES_EXCEPTIONS": False,
})
tasks.app = self.app # monkey-patch the app object
def setUp(self):
"""
Sets up the parameters for the RabbitMQ workers, and also the workers
themselves. Generates all the queues that should be in place for testing
the RabbitMQ workers.
:return: no return
"""
unittest.TestCase.setUp(self)
self.proj_home = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..'))
self._app = tasks.app
self.app = app.ADSFulltextCelery('test', proj_home=self.proj_home, local_config=\
{
'FULLTEXT_EXTRACT_PATH': os.path.join(self.proj_home, 'tests/test_unit/stub_data'),
'CELERY_BROKER': tasks.app.conf['CELERY_BROKER'] + '_test'
})
tasks.app = self.app # monkey-patch the app object
# Build the link files
build_links(test_name='integration')
self.meta_path = ''
self.channel_list = None
def setUp(self):
unittest.TestCase.setUp(self)
self.proj_home = os.path.realpath(
os.path.join(os.path.dirname(__file__), '../..'))
self._app = tasks.app
self.app = app.ADSFulltextCelery('test', proj_home=self.proj_home, local_config=\
{
'FULLTEXT_EXTRACT_PATH': os.path.join(self.proj_home, 'tests/test_unit/stub_data'),
'CELERY_BROKER': tasks.app.conf['CELERY_BROKER'] + '_test'
})
tasks.app = self.app # monkey-patch the app object
build_links(test_name='integration')
PROJ_HOME = self.app.conf['PROJ_HOME']
self.test_file = \
os.path.join(PROJ_HOME,
'tests/test_integration/stub_data/fulltext.links')
self.test_file_stub = \
os.path.join(PROJ_HOME,
'tests/test_integration/stub_data/fulltext_stub.links')
self.test_file_wrong = \
os.path.join(PROJ_HOME,
'tests/test_integration/stub_data/fulltext_wrong.links')
self.test_file_exists = \
os.path.join(PROJ_HOME,
'tests/test_integration/stub_data/fulltext_exists.links')
self.test_single_document =\
os.path.join(PROJ_HOME,
'tests/test_integration/stub_data/fulltext_single_document'
'.links')
self.test_stub_xml = \
os.path.join(PROJ_HOME,
'tests/test_unit/stub_data/test.xml')
self.test_stub_nonstandard_xml = \
os.path.join(PROJ_HOME,
'tests/test_unit/stub_data/test_failure.xml')
self.test_stub_exml = \
os.path.join(PROJ_HOME,
'tests/test_unit/stub_data/test_elsevier.xml')
self.test_stub_teixml = \
os.path.join(PROJ_HOME,
'tests/test_unit/stub_data/test.astro-ph-0002105.tei.xml')
self.test_stub_iso8859 = \
os.path.join(PROJ_HOME,
'tests/test_unit/stub_data/test.stmp_2_1_014010.iop.xml')
self.test_multi_file = os.path.join(PROJ_HOME, 'tests/test_unit/stub_data/test.xml') + \
',' + os.path.join(PROJ_HOME,'tests/test_unit/stub_data/test.xml')
self.test_stub_html = \
os.path.join(PROJ_HOME,
'tests/test_unit/stub_data/test.html')
self.test_stub_html_table = \
os.path.join(PROJ_HOME,
'tests/test_unit/stub_data/test_table.html')
self.test_stub_text = \
os.path.join(PROJ_HOME,
'tests/test_unit/stub_data/test.txt')
self.test_stub_ocr = \
os.path.join(PROJ_HOME,
'tests/test_unit/stub_data/test.ocr')
self.test_functional_stub =\
os.path.join(PROJ_HOME,
'tests/test_functional/stub_data/fulltext_functional_tests'
'.links')
if sys.version_info > (3, ):
from builtins import zip
from builtins import str
from adsputils import get_date, exceptions
import adsft.app as app_module
from kombu import Queue
from adsft import extraction, checker, writer, reader, ner
from adsmsg import FulltextUpdate
import os
from adsft.utils import TextCleaner
# ============================= INITIALIZATION ==================================== #
proj_home = os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))
app = app_module.ADSFulltextCelery('ads-fulltext',
proj_home=proj_home,
local_config=globals().get(
'local_config', {}))
logger = app.logger
app.conf.CELERY_QUEUES = (
Queue('check-if-extract', app.exchange, routing_key='check-if-extract'),
Queue('extract', app.exchange, routing_key='extract'),
Queue('extract-grobid', app.exchange, routing_key='extract-grobid'),
Queue('output-results', app.exchange, routing_key='output-results'),
Queue('facility-ner', app.exchange, routing_key='facility-ner'),
)
logger.debug("Loading spacy models for facilities...")
model1 = ner.load_model(app.conf['NER_FACILITY_MODEL_ACK'])
model2 = ner.load_model(app.conf['NER_FACILITY_MODEL_FT'])
from __future__ import absolute_import, unicode_literals
from adsputils import get_date, exceptions
import adsft.app as app_module
from kombu import Queue
from adsft import extraction, checker, writer
from adsmsg import FulltextUpdate
import os
from adsft.utils import TextCleaner
# ============================= INITIALIZATION ==================================== #
proj_home = os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))
app = app_module.ADSFulltextCelery('ads-fulltext', proj_home=proj_home)
logger = app.logger
app.conf.CELERY_QUEUES = (
Queue('check-if-extract', app.exchange, routing_key='check-if-extract'),
Queue('extract', app.exchange, routing_key='extract'),
Queue('extract-grobid', app.exchange, routing_key='extract-grobid'),
Queue('output-results', app.exchange, routing_key='output-results'),
)
# ============================= TASKS ============================================= #
@app.task(queue='check-if-extract')
def task_check_if_extract(message):
"""
Checks if the file needs to be extracted and pushes to the correct
extraction queue.
"""