def __init__(self, db, es_config=None):
self.sm_config = SMConfig.get_conf()
if not es_config:
es_config = self.sm_config['elasticsearch']
self._es = init_es_conn(es_config)
self._db = db
self.index = es_config['index']
def _create_dataset_manager(db):
config = SMConfig.get_conf()
img_store = ImageStoreServiceWrapper(config['services']['img_service_url'])
img_store.storage_type = 'fs'
return SMapiDatasetManager(db=db, es=ESExporter(db), image_store=img_store, mode='queue',
action_queue=_create_queue_publisher(SM_ANNOTATE),
status_queue=_create_queue_publisher(SM_DS_STATUS))
def __init__(self, db, es_config=None):
self.sm_config = SMConfig.get_conf()
if not es_config:
es_config = self.sm_config['elasticsearch']
self._es = init_es_conn(es_config)
self._db = db
self.index = es_config['index']
def __init__(self, qdesc, dataset_manager_factory, poll_interval=1):
self._dataset_manager_factory = dataset_manager_factory
self._sm_config = SMConfig.get_conf()
self._stopped = False
self._action_queue_consumer = QueueConsumer(config=self._sm_config['rabbitmq'], qdesc=qdesc,
callback=self._callback,
on_success=self._on_success,
on_failure=self._on_failure,
logger_name='daemon', poll_interval=poll_interval)
def __init__(self, sc, ds, ds_reader, mol_db, fdr, ds_config):
self._sc = sc
self._ds = ds
self._ds_reader = ds_reader
self._mol_db = mol_db
self._fdr = fdr
self.ds_config = ds_config
self.metrics = None
self.sm_config = SMConfig.get_conf()
def _post_to_slack(self, emoji, msg):
slack_conf = SMConfig.get_conf().get('slack', {})
if slack_conf.get('webhook_url', None):
m = {"channel": slack_conf['channel'],
"username": "******",
"text": ":{}:{}".format(emoji, msg),
"icon_emoji": ":robot_face:"}
post(slack_conf['webhook_url'], json=m)
def _save_data_from_raw_ms_file(self):
ms_file_type_config = SMConfig.get_ms_file_handler(self._wd_manager.local_dir.ms_file_path)
acq_geometry_factory_module = ms_file_type_config['acq_geometry_factory']
acq_geometry_factory = getattr(import_module(acq_geometry_factory_module['path']),
acq_geometry_factory_module['name'])
acq_geometry = acq_geometry_factory(self._wd_manager.local_dir.ms_file_path).create()
self._ds.save_acq_geometry(self._db, acq_geometry)
self._ds.save_ion_img_storage_type(self._db, ms_file_type_config['img_storage_type'])
def _save_data_from_raw_ms_file(self):
ms_file_type_config = SMConfig.get_ms_file_handler(self._wd_manager.local_dir.ms_file_path)
acq_geometry_factory_module = ms_file_type_config['acq_geometry_factory']
acq_geometry_factory = getattr(import_module(acq_geometry_factory_module['path']),
acq_geometry_factory_module['name'])
acq_geometry = acq_geometry_factory(self._wd_manager.local_dir.ms_file_path).create()
self._ds.save_acq_geometry(self._db, acq_geometry)
self._ds.save_ion_img_storage_type(self._db, ms_file_type_config['img_storage_type'])
def _create_dataset_manager(db):
config = SMConfig.get_conf()
img_store = ImageStoreServiceWrapper(config['services']['img_service_url'])
img_store.storage_type = 'fs'
return SMapiDatasetManager(
db=db,
es=ESExporter(db),
image_store=img_store,
mode='queue',
action_queue=_create_queue_publisher(SM_ANNOTATE),
status_queue=_create_queue_publisher(SM_DS_STATUS))
def _post_to_slack(self, emoji, msg):
slack_conf = SMConfig.get_conf().get('slack', {})
if slack_conf.get('webhook_url', None):
m = {
"channel": slack_conf['channel'],
"username": "******",
"text": ":{}:{}".format(emoji, msg),
"icon_emoji": ":robot_face:"
}
post(slack_conf['webhook_url'], json=m)
def __init__(self, sm_config_path):
self.sm_config_path = sm_config_path
self.sm_config = SMConfig.get_conf()
self.db = DB(self.sm_config['db'])
self.ds_id = '2000-01-01-00_00_00'
self.base_search_res_path = join(proj_root(), 'tests/reports', 'spheroid_untreated_search_res.csv')
self.ds_name = 'sci_test_spheroid_untreated'
self.data_dir_path = join(self.sm_config['fs']['base_path'], self.ds_name)
self.input_path = join(proj_root(), 'tests/data/untreated')
self.ds_config_path = join(self.input_path, 'config.json')
self.metrics = ['chaos', 'spatial', 'spectral']
def __init__(self, qdesc, dataset_manager_factory, poll_interval=1):
self._dataset_manager_factory = dataset_manager_factory
self._sm_config = SMConfig.get_conf()
self._stopped = False
self._action_queue_consumer = QueueConsumer(
config=self._sm_config['rabbitmq'],
qdesc=qdesc,
callback=self._callback,
on_success=self._on_success,
on_failure=self._on_failure,
logger_name='daemon',
poll_interval=poll_interval)
def __init__(self, sc, name, owner_email, input_path, ds_config,
wd_manager, db):
self.db = db
self.sc = sc
self.name = name
self.owner_email = owner_email
self.input_path = input_path
self.ds_config = ds_config
self.wd_manager = wd_manager
self.sm_config = SMConfig.get_conf()
self._define_pixels_order()
def __init__(self, ds_name, imzml_path, txt_path, coord_path=None):
self.ds_name = ds_name
self.imzml_path = imzml_path
self.preprocess = None
self.sm_config = SMConfig.get_conf()
self.coord_set = set()
self.txt_path = txt_path
self.coord_path = coord_path
self.txt_file = None
self.coord_file = None
self.parser = None
def __init__(self, client_email, ds_name):
self.sm_config = SMConfig.get_conf()
self.client_email = client_email
self.ds_name = ds_name
self.ds_id = None
self.job_id = None
self.sc = None
self.db = None
self.ds = None
self.fdr = None
self.formulas = None
self.ds_config = None
self.wd_manager = None
def __init__(self, ds_name, imzml_path, txt_path, coord_path=None):
self.ds_name = ds_name
self.imzml_path = imzml_path
self.preprocess = None
self.sm_config = SMConfig.get_conf()
self.coord_set = set()
self.txt_path = txt_path
self.coord_path = coord_path
self.txt_file = None
self.coord_file = None
self.parser = None
def __init__(self,
db=None,
es=None,
img_store=None,
mode=None,
status_queue=None,
logger_name=None):
self._sm_config = SMConfig.get_conf()
self._db = db
self._es = es
self._img_store = img_store
self.mode = mode
self._status_queue = status_queue
self.logger = logging.getLogger(logger_name)
def __init__(self,
db,
es,
img_store,
status_queue=None,
logger=None,
sm_config=None):
self._sm_config = sm_config or SMConfig.get_conf()
self._slack_conf = self._sm_config.get('slack', {})
self._db = db
self.es = es
self._img_store = img_store
self.status_queue = status_queue
self.logger = logger or logging.getLogger()
def __init__(self, sm_config_path):
self.sm_config_path = sm_config_path
self.sm_config = SMConfig.get_conf()
self.db = DB(self.sm_config['db'])
self.ds_id = '2000-01-01-00_00_00'
self.base_search_res_path = join(proj_root(), 'tests/reports',
'spheroid_untreated_search_res.csv')
self.ds_name = 'sci_test_spheroid_untreated'
self.data_dir_path = join(self.sm_config['fs']['base_path'],
self.ds_name)
self.input_path = join(proj_root(), 'tests/data/untreated')
self.ds_config_path = join(self.input_path, 'config.json')
self.metrics = ['chaos', 'spatial', 'spectral']
def __init__(self, sc, moldb_name, isocalc):
self._sc = sc
self._moldb_name = moldb_name
self._isocalc = isocalc
self._sm_config = SMConfig.get_conf()
self._parquet_chunks_n = 64
self._iso_gen_part_n = 512
self._spark_session = SparkSession(self._sc)
self._ion_centroids_path = '{}/{}/{}/{}'.format(
self._sm_config['isotope_storage']['path'], self._moldb_name,
self._isocalc.sigma, self._isocalc.charge)
self.ion_df = None
self.ion_centroids_df = None
def __init__(self, sc, moldb_name, isocalc):
self._sc = sc
self._moldb_name = moldb_name
self._isocalc = isocalc
self._sm_config = SMConfig.get_conf()
self._parquet_chunks_n = 64
self._iso_gen_part_n = 512
self._spark_session = SparkSession(self._sc)
self._ion_centroids_path = '{}/{}/{}/{}'.format(self._sm_config['isotope_storage']['path'],
self._moldb_name,
self._isocalc.sigma,
self._isocalc.charge)
self.ion_df = None
self.ion_centroids_df = None
def __init__(self,
db,
es,
image_store,
logger=None,
annot_queue=None,
update_queue=None,
status_queue=None):
self._sm_config = SMConfig.get_conf()
self._db = db
self._es = es
self._img_store = image_store
self._status_queue = status_queue
self._annot_queue = annot_queue
self._update_queue = update_queue
self.logger = logger or logging.getLogger()
def __init__(self, ds_name):
self.sm_config = SMConfig.get_conf()
if 's3_base_path' not in self.sm_config['fs']:
self.local_fs_only = True
elif not self.sm_config['fs']['s3_base_path']:
self.local_fs_only = True
else:
self.local_fs_only = False
self.s3 = boto3.session.Session().resource('s3')
self.s3transfer = S3Transfer(boto3.client('s3', 'eu-west-1'))
self.local_dir = LocalWorkDir(self.sm_config['fs']['base_path'], ds_name)
if not self.local_fs_only:
self.remote_dir = S3WorkDir(self.sm_config['fs']['s3_base_path'], ds_name, self.s3, self.s3transfer)
def __init__(self, manager, update_qdesc, poll_interval=1):
self._manager = manager
self._sm_config = SMConfig.get_conf()
self._db = DB(self._sm_config['db'])
self._update_queue_cons = QueueConsumer(
config=self._sm_config['rabbitmq'],
qdesc=update_qdesc,
callback=self._callback,
on_success=self._on_success,
on_failure=self._on_failure,
logger=self.logger,
poll_interval=poll_interval)
self._status_queue_pub = QueuePublisher(
config=self._sm_config['rabbitmq'],
qdesc=SM_DS_STATUS,
logger=self.logger)
self._stopped = False
def __init__(self, img_store=None, no_clean=False):
self.no_clean = no_clean
self._img_store = img_store
self._job_id = None
self._sc = None
self._db = None
self._ds = None
self._ds_reader = None
self._status_queue = None
self._fdr = None
self._wd_manager = None
self._es = None
self._sm_config = SMConfig.get_conf()
logger.debug('Using SM config:\n%s', pformat(self._sm_config))
def __init__(self, img_store=None, no_clean=False):
self.no_clean = no_clean
self._img_store = img_store
self._job_id = None
self._sc = None
self._db = None
self._ds = None
self._ds_reader = None
self._status_queue = None
self._fdr = None
self._wd_manager = None
self._es = None
self._sm_config = SMConfig.get_conf()
logger.debug('Using SM config:\n%s', pformat(self._sm_config))
def ms_file_path(self):
if self._ms_file_path:
return self._ms_file_path
file_handlers = SMConfig.get_conf()['ms_file_handlers']
for handler in file_handlers:
ms_file_extension = handler['extensions'][0]
logger.info('"%s" file handler is looking for files with "%s" extension \
in the input directory', handler['type'], ms_file_extension)
ms_file_path = next((fn for fn in listdir(self.ds_path) \
if re.search(r'\.{}$'.format(ms_file_extension), fn, re.IGNORECASE)), None)
if ms_file_path:
logger.info('"%s" file handler has found "%s" in the input directory',
handler['type'], ms_file_path)
self._ms_file_path = join(self.ds_path, ms_file_path)
break
return self._ms_file_path if self._ms_file_path else ''
def __init__(self, ds_name):
self.sm_config = SMConfig.get_conf()
if 's3_base_path' not in self.sm_config['fs']:
self.local_fs_only = True
elif not self.sm_config['fs']['s3_base_path']:
self.local_fs_only = True
else:
self.local_fs_only = False
self.s3 = boto3.session.Session().resource('s3')
self.s3transfer = S3Transfer(boto3.client('s3', 'eu-west-1'))
self.local_dir = LocalWorkDir(self.sm_config['fs']['base_path'],
ds_name)
if not self.local_fs_only:
self.remote_dir = S3WorkDir(self.sm_config['fs']['s3_base_path'],
ds_name, self.s3, self.s3transfer)
def reindex_results(ds_id, ds_mask):
assert ds_id or ds_mask
conf = SMConfig.get_conf()
if ds_mask == '_all_':
_reindex_all(conf)
else:
db = DB(conf['db'])
es_exp = ESExporter(db)
if ds_id:
rows = db.select("select id, name, config from dataset where id = '{}'".format(ds_id))
elif ds_mask:
rows = db.select("select id, name, config from dataset where name like '{}%'".format(ds_mask))
else:
rows = []
_reindex_datasets(rows, es_exp)
def __init__(self, ds_id):
self.sm_config = SMConfig.get_conf()
if not self.sm_config['fs'].get('s3_base_path', None):
self.local_fs_only = True
elif not self.sm_config['fs']['s3_base_path']:
self.local_fs_only = True
else:
self.local_fs_only = False
cred_dict = dict(aws_access_key_id=self.sm_config['aws']['aws_access_key_id'],
aws_secret_access_key=self.sm_config['aws']['aws_secret_access_key'])
session = boto3.session.Session(**cred_dict)
self.s3 = session.resource('s3')
self.s3transfer = S3Transfer(boto3.client('s3', 'eu-west-1', **cred_dict))
self.local_dir = LocalWorkDir(self.sm_config['fs']['base_path'], ds_id)
if not self.local_fs_only:
self.remote_dir = S3WorkDir(self.sm_config['fs']['s3_base_path'], ds_id, self.s3, self.s3transfer)
def __init__(self, id=None, name=None, version=None, iso_gen_config=None,
mol_db_service=None, db=None):
self._iso_gen_config = iso_gen_config
sm_config = SMConfig.get_conf()
self._mol_db_service = mol_db_service or MolDBServiceWrapper(sm_config['services']['mol_db'])
self._db = db
if id is not None:
data = self._mol_db_service.find_db_by_id(id)
elif name is not None:
data = self._mol_db_service.find_db_by_name_version(name, version)[0]
else:
raise Exception('MolDB id or name should be provided')
self._id, self._name, self._version = data['id'], data['name'], data['version']
self._sf_df = None
self._job_id = None
self._sfs = None
self._ion_centroids = None
def reindex_results(ds_id, ds_mask):
assert ds_id or ds_mask
conf = SMConfig.get_conf()
if ds_mask == '_all_':
_reindex_all(conf)
else:
db = DB(conf['db'])
es_exp = ESExporter(db)
if ds_id:
rows = db.select(
"select id, name, config from dataset where id = '{}'".format(
ds_id))
elif ds_mask:
rows = db.select(
"select id, name, config from dataset where name like '{}%'".
format(ds_mask))
else:
rows = []
_reindex_datasets(rows, es_exp)
from sm.engine.util import SMConfig, logger, sm_log_formatters
from sm.engine.search_job import SearchJob
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='SM process dataset at a remote spark location.')
parser.add_argument('ds_name', type=str, help='Dataset name')
parser.add_argument('input_path', type=str, help='Path to a dataset location')
parser.add_argument('ds_config_path', type=str, help='Path to a dataset config file')
parser.add_argument('--config', dest='sm_config_path', type=str, help='SM config path')
parser.add_argument('--no-clean', dest='no_clean', action='store_true', help='do not clean interim files')
start = time.time()
args = parser.parse_args()
SMConfig.set_path(args.sm_config_path)
fileHandler = FileHandler(filename='logs/{}.log'.format(args.ds_name.replace('/', '_')))
fileHandler.setLevel(DEBUG)
fileHandler.setFormatter(Formatter(sm_log_formatters['SM']['format']))
logger.addHandler(fileHandler)
logger.debug('Using SM config:\n%s', pformat(SMConfig.get_conf()))
logger.info("Processing...")
job = SearchJob(None, args.ds_name)
job.run(args.input_path, args.ds_config_path, clean=not args.no_clean)
logger.info("All done!")
time_spent = time.time() - start
"JOIN agg_formula f ON f.id = m.sf_id AND sf_db.id = f.db_id "
"JOIN job j ON j.id = m.job_id "
"JOIN dataset ds ON ds.id = j.ds_id "
"JOIN theor_peaks tp ON tp.db_id = sf_db.id AND tp.sf_id = m.sf_id AND tp.adduct = m.adduct "
"WHERE sf_db.name = %s AND ds.name = %s "
"AND ROUND(sigma::numeric, 6) = %s AND charge = %s AND pts_per_mz = %s")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Exporting search results into a csv file')
parser.add_argument('ds_name', type=str, help='Dataset name')
parser.add_argument('csv_path', type=str, help='Path for the csv file')
parser.add_argument('--config', dest='sm_config_path', type=str, help='SM config path')
parser.set_defaults(sm_config_path=path.join(proj_root(), 'conf/config.json'))
args = parser.parse_args()
SMConfig.set_path(args.sm_config_path)
db = DB(SMConfig.get_conf()['db'])
ds_config = db.select_one(DS_CONFIG_SEL, args.ds_name)[0]
isotope_gen_config = ds_config['isotope_generation']
charge = '{}{}'.format(isotope_gen_config['charge']['polarity'], isotope_gen_config['charge']['n_charges'])
export_rs = db.select(EXPORT_SEL, ds_config['database']['name'], args.ds_name,
isotope_gen_config['isocalc_sigma'], charge, isotope_gen_config['isocalc_pts_per_mz'])
header = ','.join(['formula_db', 'ds_name', 'sf', 'adduct', 'chaos', 'img_corr', 'pat_match',
'isocalc_sigma', 'isocalc_charge', 'isocalc_pts_per_mz', 'first_peak_mz']) + '\n'
with open(args.csv_path, 'w') as f:
f.write(header)
f.writelines([','.join(map(str, row)) + '\n' for row in export_rs])
logger.info('Exported all search results for "%s" dataset into "%s" file', args.ds_name, args.csv_path)
def __init__(self, es_config=None):
if not es_config:
es_config = SMConfig.get_conf()['elasticsearch']
self._es = init_es_conn(es_config)
self._ind_client = IndicesClient(self._es)
from pprint import pprint
from subprocess import check_call
import numpy as np
from fabric.api import local
from fabric.context_managers import warn_only
from sm.engine.db import DB
from sm.engine.util import proj_root, SMConfig
# def sm_config():
# with open(join(proj_root(), 'conf/config.json')) as f:
# return json.load(f)
SMConfig.set_path(join(proj_root(), 'conf/config.json'))
sm_config = SMConfig.get_conf()
ds_name = 'sci_test_spheroid_12h'
data_dir_path = join(SMConfig.get_conf()['fs']['base_path'], ds_name)
input_dir_path = join(proj_root(), 'test/data/sci_test_search_job_spheroid_dataset')
ds_config_path = join(input_dir_path, 'config.json')
SEARCH_RES_SELECT = ("select sf, adduct, stats "
"from iso_image_metrics s "
"join formula_db sf_db on sf_db.id = s.db_id "
"join agg_formula f on f.id = s.sf_id AND sf_db.id = f.db_id "
"join job j on j.id = s.job_id "
"join dataset ds on ds.id = j.ds_id "
"where ds.name = %s and sf_db.name = %s "
"ORDER BY sf, adduct ")
def _init_ms_parser_factory(self):
ms_file_type_config = SMConfig.get_ms_file_handler(self.ms_file_path)
ms_parser_factory_module = ms_file_type_config['parser_factory']
self._parser_factory = getattr(import_module(ms_parser_factory_module['path']), ms_parser_factory_module['name'])
def _create_queue_publisher(qdesc):
config = SMConfig.get_conf()
return QueuePublisher(config['rabbitmq'], qdesc, logger)
action='store_true',
help='compare current search results with previous')
parser.add_argument('-s',
'--save',
action='store_true',
help='store current search results')
parser.add_argument('--config',
dest='sm_config_path',
default=join(proj_root(), 'conf/config.json'),
help='path to sm config file')
parser.add_argument('--mock-img-store',
action='store_true',
help='whether to mock the Image Store Service')
args = parser.parse_args()
SMConfig.set_path(args.sm_config_path)
init_loggers(SMConfig.get_conf()['logs'])
sci_tester = SciTester(args.sm_config_path)
if args.run:
run_search_successful = False
search_results_different = False
try:
sci_tester.run_search(args.mock_img_store)
run_search_successful = True
search_results_different = sci_tester.search_results_are_different(
)
except Exception as e:
if not run_search_successful:
raise Exception('Search was not successful!') from e
if transform != None:
ds = api.Dataset.load(db=db, ds_id=id)
img_store = ImageStoreServiceWrapper(
config['services']['img_service_url'])
img_store.storage_type = 'fs'
sm = SMapiDatasetManager(db=db,
es=ESExporter(db),
image_store=img_store,
mode='queue')
ds_opt_img_query = 'SELECT optical_image from dataset {}'.format(
'WHERE id = %s')
img_id = db.select(ds_opt_img_query, params=(ds.id, ))
sm._add_thumbnail_optical_image(ds, f"{img_id[0][0]}", transform)
SMConfig.set_path('conf/config.json')
sm_config = SMConfig.get_conf()
set_metadata_thumbnail(DB(sm_config['db']), sm_config, 'Untreated_3_434')
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Updates thumbnail for a provided dataset")
parser.add_argument('--ds-name',
dest='ds_name',
type=str,
help="Process specific dataset given by its name")
parser.add_argument('--config',
dest='sm_config_path',
default='conf/config.json',
type=str,
help='SM config path')
@post('/v1/datasets/<ds_id>/delete')
@sm_modify_dataset('DELETE')
def delete_ds(ds_man, ds, params):
del_raw = params.get('del_raw', False)
ds_man.delete(ds, del_raw_data=del_raw)
@post('/v1/datasets/<ds_id>/add-optical-image')
@sm_modify_dataset('ADD_OPTICAL_IMAGE')
def add_optical_image(ds_man, ds, params):
img_id = params['url'].split('/')[-1]
ds_man.add_optical_image(ds, img_id, params['transform'])
@post('/v1/datasets/<ds_id>/del-optical-image')
@sm_modify_dataset('DEL_OPTICAL_IMAGE')
def del_optical_image(ds_man, ds, params):
ds_man.del_optical_image(ds)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='SM Engine REST API')
parser.add_argument('--config', dest='config_path', default='conf/config.json', type=str, help='SM config path')
args = parser.parse_args()
SMConfig.set_path(args.config_path)
init_loggers(SMConfig.get_conf()['logs'])
logger = logging.getLogger(name='api')
run(**SMConfig.get_conf()['bottle'])
def _create_db_conn():
config = SMConfig.get_conf()
return DB(config['db'])
rows = db.select(
"select id, name, config from dataset where id = '{}'".format(
ds_id))
elif ds_mask:
rows = db.select(
"select id, name, config from dataset where name like '{}%'".
format(ds_mask))
else:
rows = []
_reindex_datasets(rows, es_exp)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Reindex dataset results')
parser.add_argument('--config',
default='conf/config.json',
help='SM config path')
parser.add_argument('--ds-id', dest='ds_id', default='', help='DS id')
parser.add_argument('--ds-name',
dest='ds_name',
default='',
help='DS name prefix mask (_all_ for all datasets)')
args = parser.parse_args()
SMConfig.set_path(args.config)
init_loggers(SMConfig.get_conf()['logs'])
logger = logging.getLogger('engine')
reindex_results(args.ds_id, args.ds_name)
conf = SMConfig.get_conf()
if ds_mask == '_all_':
_reindex_all(conf)
else:
db = DB(conf['db'])
es_exp = ESExporter(db)
if ds_id:
rows = db.select("select id, name, config from dataset where id = '{}'".format(ds_id))
elif ds_mask:
rows = db.select("select id, name, config from dataset where name like '{}%'".format(ds_mask))
else:
rows = []
_reindex_datasets(rows, es_exp)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Reindex dataset results')
parser.add_argument('--config', default='conf/config.json', help='SM config path')
parser.add_argument('--ds-id', dest='ds_id', default='', help='DS id')
parser.add_argument('--ds-name', dest='ds_name', default='', help='DS name prefix mask (_all_ for all datasets)')
args = parser.parse_args()
SMConfig.set_path(args.config)
init_loggers(SMConfig.get_conf()['logs'])
logger = logging.getLogger('engine')
reindex_results(args.ds_id, args.ds_name)
def _create_queue_publisher(qdesc):
config = SMConfig.get_conf()
return QueuePublisher(config['rabbitmq'], qdesc, logger)
UPD_DATASET_THUMB_OPTICAL_IMAGE = 'update dataset set thumbnail = %s WHERE id = %s'
SEL_OPTICAL_IMAGE_THUMBNAIL = 'SELECT thumbnail FROM dataset WHERE id = %s'
def set_metadata_thumbnail(db, config, ds_name):
ds_thumb_query = 'SELECT id, transform, thumbnail from dataset {}'.format('WHERE name = %s' if ds_name != ALL_DS_MASK else '')
for id, transform, thumbnail in db.select(ds_thumb_query, params=(ds_name,) if ds_name else None):
if transform != None:
ds = api.Dataset.load(db=db, ds_id=id)
img_store = ImageStoreServiceWrapper(config['services']['img_service_url'])
img_store.storage_type = 'fs'
sm = SMapiDatasetManager(db=db, es=ESExporter(db), image_store=img_store, mode='queue')
ds_opt_img_query = 'SELECT optical_image from dataset {}'.format('WHERE id = %s')
img_id = db.select(ds_opt_img_query, params=(ds.id,))
sm._add_thumbnail_optical_image(ds, f"{img_id[0][0]}", transform)
SMConfig.set_path('conf/config.json')
sm_config = SMConfig.get_conf()
set_metadata_thumbnail(DB(sm_config['db']), sm_config, 'Untreated_3_434')
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Updates thumbnail for a provided dataset")
parser.add_argument('--ds-name', dest='ds_name', type=str, help="Process specific dataset given by its name")
parser.add_argument('--config', dest='sm_config_path', default='conf/config.json', type=str, help='SM config path')
args = parser.parse_args()
SMConfig.set_path(args.sm_config_path)
sm_config = SMConfig.get_conf()
db = DB(sm_config['db'])
if args.ds_name:
"AND ROUND(sigma::numeric, 6) = %s AND charge = %s AND pts_per_mz = %s")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Exporting search results into a csv file')
parser.add_argument('ds_name', type=str, help='Dataset name')
parser.add_argument('csv_path', type=str, help='Path for the csv file')
parser.add_argument('--config',
dest='sm_config_path',
type=str,
help='SM config path')
parser.set_defaults(
sm_config_path=path.join(proj_root(), 'conf/config.json'))
args = parser.parse_args()
SMConfig.set_path(args.sm_config_path)
db = DB(SMConfig.get_conf()['db'])
ds_config = db.select_one(DS_CONFIG_SEL, args.ds_name)[0]
isotope_gen_config = ds_config['isotope_generation']
charge = '{}{}'.format(isotope_gen_config['charge']['polarity'],
isotope_gen_config['charge']['n_charges'])
export_rs = db.select(EXPORT_SEL, ds_config['database']['name'],
args.ds_name, isotope_gen_config['isocalc_sigma'],
charge, isotope_gen_config['isocalc_pts_per_mz'])
header = '\t'.join(['formula_db', 'db_ids', 'sf_name', 'sf', 'adduct']) +'\t' + '\t'.join(metrics) + '\t' + \
'\t'.join(['fdr', 'isocalc_sigma', 'isocalc_charge', 'isocalc_pts_per_mz', 'first_peak_mz']) + '\n'
with open(args.csv_path, 'w') as f:
f.write(header)
f.writelines(['\t'.join(map(str, row)) + '\n' for row in export_rs])
ds_man.delete(ds, del_raw_data=del_raw, force=force)
@post('/v1/datasets/<ds_id>/add-optical-image')
@sm_modify_dataset('ADD_OPTICAL_IMAGE')
def add_optical_image(ds_man, ds, params):
img_id = params['url'].split('/')[-1]
ds_man.add_optical_image(ds, img_id, params['transform'])
@post('/v1/datasets/<ds_id>/del-optical-image')
@sm_modify_dataset('DEL_OPTICAL_IMAGE')
def del_optical_image(ds_man, ds, params):
ds_man.del_optical_image(ds)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='SM Engine REST API')
parser.add_argument('--config',
dest='config_path',
default='conf/config.json',
type=str,
help='SM config path')
args = parser.parse_args()
SMConfig.set_path(args.config_path)
init_loggers(SMConfig.get_conf()['logs'])
logger = logging.getLogger(name='api')
logger.info('Starting SM api')
run(**SMConfig.get_conf()['bottle'])
def __init__(self, es_config=None):
if not es_config:
es_config = SMConfig.get_conf()['elasticsearch']
self._es = init_es_conn(es_config)
self._ind_client = IndicesClient(self._es)
from sm.engine.util import SMConfig, init_loggers, create_ds_from_files
from sm.engine.search_job import SearchJob
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='SM process dataset at a remote spark location.')
parser.add_argument('--ds-id', dest='ds_id', type=str, help='Dataset id')
parser.add_argument('--ds-name', dest='ds_name', type=str, help='Dataset name')
parser.add_argument('--input-path', type=str, help='Path to a dataset location')
parser.add_argument('--no-clean', dest='no_clean', action='store_true',
help="Don't clean dataset txt files after job is finished")
parser.add_argument('--config', dest='sm_config_path', default='conf/config.json',
type=str, help='SM config path')
args = parser.parse_args()
SMConfig.set_path(args.sm_config_path)
sm_config = SMConfig.get_conf()
init_loggers(sm_config['logs'])
db = DB(sm_config['db'])
img_store = ImageStoreServiceWrapper(sm_config['services']['img_service_url'])
ds_man = SMDaemonDatasetManager(db, ESExporter(db), img_store, mode='local')
try:
ds = create_ds_from_files(args.ds_id, args.ds_name, args.input_path)
ds_man.add(ds, SearchJob, del_first=True)
except Exception as e:
logging.getLogger('engine').error(e)
sys.exit(1)
sys.exit()
def _create_db_conn():
config = SMConfig.get_conf()
return DB(config['db'])
def clear_data_dirs(self):
with warn_only():
local('rm -rf {}'.format(self.data_dir_path))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Scientific tests runner')
parser.add_argument('-r', '--run', action='store_true', help='compare current search results with previous')
parser.add_argument('-s', '--save', action='store_true', help='store current search results')
parser.add_argument('--config', dest='sm_config_path',
default=join(proj_root(), 'conf/config.json'),
help='path to sm config file')
parser.add_argument('--mock-img-store', action='store_true', help='whether to mock the Image Store Service')
args = parser.parse_args()
SMConfig.set_path(args.sm_config_path)
init_loggers(SMConfig.get_conf()['logs'])
sci_tester = SciTester(args.sm_config_path)
if args.run:
run_search_successful = False
search_results_different = False
try:
sci_tester.run_search(args.mock_img_store)
run_search_successful = True
search_results_different = sci_tester.search_results_are_different()
except Exception as e:
if not run_search_successful:
raise Exception('Search was not successful!') from e
elif search_results_different:
import argparse
from sm.engine.util import SMConfig, init_loggers
from sm.engine import ESIndexManager
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Create ElasticSearch indices')
parser.add_argument('--config', dest='config_path', default='conf/config.json', help='SM config path')
parser.add_argument('--drop', action='store_true', help='Delete index if exists')
args = parser.parse_args()
SMConfig.set_path(args.config_path)
init_loggers(SMConfig.get_conf()['logs'])
es_config = SMConfig.get_conf()['elasticsearch']
es_man = ESIndexManager(es_config)
alias = es_config['index']
index = es_man.internal_index_name(alias)
if args.drop:
es_man.delete_index(index)
es_man.create_index(index)
es_man.remap_alias(index, alias)
def _fetch_ds_metadata(self, ds_id):
db = DB(SMConfig.get_conf()['db'])
res = db.select_one('SELECT name, metadata FROM dataset WHERE id = %s', params=(ds_id,))
return res or ('', {})
def setupQueue(sm_config_path):
SMConfig.set_path(sm_config_path)
rabbitmq_config = SMConfig.get_conf()['rabbitmq']
return QueuePublisher(rabbitmq_config, 'sm_annotate')
from sm.engine.util import SMConfig, init_loggers, create_ds_from_files
from sm.engine.search_job import SearchJob
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='SM process dataset at a remote spark location.')
parser.add_argument('--ds-id', dest='ds_id', type=str, help='Dataset id')
parser.add_argument('--ds-name', dest='ds_name', type=str, help='Dataset name')
parser.add_argument('--input-path', type=str, help='Path to a dataset location')
parser.add_argument('--no-clean', dest='no_clean', action='store_true',
help="Don't clean dataset txt files after job is finished")
parser.add_argument('--config', dest='sm_config_path', default='conf/config.json',
type=str, help='SM config path')
args = parser.parse_args()
SMConfig.set_path(args.sm_config_path)
sm_config = SMConfig.get_conf()
init_loggers(sm_config['logs'])
db = DB(sm_config['db'])
img_store = ImageStoreServiceWrapper(sm_config['services']['img_service_url'])
ds_man = SMDaemonDatasetManager(db, ESExporter(db), img_store, mode='local')
try:
ds = create_ds_from_files(args.ds_id, args.ds_name, args.input_path)
ds_man.add(ds, SearchJob, del_first=True)
except Exception as e:
logging.getLogger('engine').error(e, exc_info=True)
sys.exit(1)
sys.exit()