def init_queue_pub(qname='annotate'):
from sm.engine import queue
queue.SM_ANNOTATE['name'] = queue.SM_ANNOTATE['name'] + '_test'
queue.SM_UPDATE['name'] = queue.SM_UPDATE['name'] + '_test'
if qname == 'annotate':
qdesc = queue.SM_ANNOTATE
elif qname == 'update':
qdesc = queue.SM_UPDATE
else:
raise Exception(f'Wrong qname={qname}')
queue_pub = queue.QueuePublisher(config=sm_config()['rabbitmq'],
qdesc=qdesc,
logger=logger)
return queue_pub
def run_daemons(db, es):
from sm.engine.queue import QueuePublisher, SM_DS_STATUS, SM_ANNOTATE, SM_UPDATE
from sm.engine.png_generator import ImageStoreServiceWrapper
from sm.engine.sm_daemons import SMDaemonManager, SMAnnotateDaemon, SMUpdateDaemon
status_queue_pub = QueuePublisher(config=sm_config()['rabbitmq'],
qdesc=SM_DS_STATUS,
logger=logger)
manager = SMDaemonManager(
db=db, es=es,
img_store=ImageStoreServiceWrapper(sm_config()['services']['img_service_url']),
status_queue=status_queue_pub,
logger=logger,
sm_config=sm_config()
)
annotate_daemon = SMAnnotateDaemon(manager=manager,
annot_qdesc=SM_ANNOTATE,
upd_qdesc=SM_UPDATE)
annotate_daemon.start()
annotate_daemon.stop()
update_daemon = SMUpdateDaemon(manager=manager,
update_qdesc=SM_UPDATE)
update_daemon.start()
update_daemon.stop()
import pytest
from fabric.api import local
from fabric.context_managers import warn_only
from mock import patch
from sm.engine.db import DB
from sm.engine.search_job import SearchJob
from sm.engine.util import SMConfig
from sm.engine.fdr import DECOY_ADDUCTS
from sm.engine.tests.util import create_test_db, drop_test_db, sm_config
test_ds_name = 'imzml_example_ds'
proj_dir_path = dirname(dirname(__file__))
data_dir_path = join(sm_config()["fs"]["base_path"], test_ds_name)
input_dir_path = join(proj_dir_path, 'tests/data/imzml_example_ds')
ds_config_path = join(input_dir_path, 'config.json')
@pytest.fixture()
def create_fill_sm_database(create_test_db, drop_test_db, sm_config):
local('psql -h localhost -U sm sm_test < {}'.format(join(proj_dir_path, 'scripts/create_schema.sql')))
db = DB(sm_config['db'])
try:
db.insert('INSERT INTO formula_db VALUES (%s, %s, %s)',
[(0, '2016-01-01', 'HMDB')])
db.insert('INSERT INTO formula VALUES (%s, %s, %s, %s, %s)',
[(100, 0, '00001', 'compound_name', 'C12H24O')])
db.insert('INSERT INTO agg_formula VALUES (%s, %s, %s, %s, %s)',
from datetime import datetime
from sm.engine.db import DB
from sm.engine.errors import SMError, JobFailedError, ESExportFailedError
from sm.engine.search_job import SearchJob
from sm.engine.fdr import DECOY_ADDUCTS
from sm.engine.dataset import Dataset
from sm.engine.dataset_manager import DatasetStatus
from sm.engine.tests.util import test_db, sm_config, sm_index, es, es_dsl_search
from sm.engine.acq_geometry_factory import ACQ_GEOMETRY_KEYS
from sm.engine.png_generator import ImageStoreServiceWrapper
test_ds_name = 'imzml_example_ds'
proj_dir_path = dirname(dirname(__file__))
data_dir_path = join(sm_config()["fs"]["base_path"], test_ds_name)
input_dir_path = join(proj_dir_path, 'tests/data/imzml_example_ds')
ds_config_path = join(input_dir_path, 'config.json')
@pytest.fixture()
def create_fill_sm_database(test_db, sm_index, sm_config):
local('psql -h localhost -U sm sm_test < {}'.format(join(proj_dir_path, 'scripts/create_schema.sql')))
@pytest.fixture()
def clean_isotope_storage(sm_config):
with warn_only():
local('rm -rf {}'.format(sm_config['isotope_storage']['path']))
import pytest
from fabric.api import local
from fabric.context_managers import warn_only
from sm.engine.db import DB
from sm.engine.es_export import ESExporter
from sm.engine.dataset import Dataset, DatasetStatus
from sm.engine.acq_geometry_factory import ACQ_GEOMETRY_KEYS
from sm.engine.search_job import JobStatus
from sm.engine.tests.util import sm_config, ds_config, test_db, init_loggers, sm_index, es_dsl_search
os.environ.setdefault('PYSPARK_PYTHON', sys.executable)
init_loggers(sm_config()['logs'])
logger = logging.getLogger('annotate-daemon')
test_ds_name = 'imzml_example_ds'
proj_dir_path = dirname(dirname(__file__))
data_dir_path = join(sm_config()["fs"]["base_path"], test_ds_name)
input_dir_path = join(proj_dir_path, 'tests/data/imzml_example_ds')
ds_config_path = join(input_dir_path, 'config.json')
@pytest.fixture()
def clean_isotope_storage(sm_config):
with warn_only():
local('rm -rf {}'.format(sm_config['isotope_storage']['path']))
from unittest.mock import MagicMock
import json
from os import listdir
from os.path import join, basename
from subprocess import check_call
import pytest
from fabric.api import local
from fabric.context_managers import lcd, warn_only
from sm.engine.util import SMConfig
from sm.engine.work_dir import WorkDirManager, S3WorkDir
from sm.engine.tests.util import sm_config
ds_id = '2000-01-01_00h00m'
input_local_path = join(sm_config()['fs']['base_path'], 'test_input_path')
input_remote_path = 's3a://somepath.com/archive'
data_dir_path = sm_config()['fs']['base_path']
ds_path = join(data_dir_path, ds_id)
def create_sample_files(path, copy_config=True):
local('mkdir -p {}'.format(path))
with lcd(path):
local(' echo "FOO!" > foo.imzML')
local(' echo "FOO!" > foo.ibd')
local(' echo "FOO!" > meta.json')
if copy_config:
with open(join(path, 'config.json'), 'w') as f:
f.write(json.dumps({}))