def parse_cmd_args (self, cmd_args):
parser = argparse.ArgumentParser()
parser.add_argument('dbpath',
help='Path to aggregator output')
parser.add_argument('-f',
dest='filterpath',
default=None,
help='Path to filter file')
parser.add_argument('-F',
dest='filtername',
default=None,
help='Name of filter (stored in aggregator output)')
parser.add_argument('--filterstring',
dest='filterstring',
default=None,
help='Filter in JSON')
parser.add_argument('-s',
dest='savepath',
default=None,
help='Path to save file')
parser.add_argument('-c',
dest='confpath',
help='path to a conf file')
parsed_args = parser.parse_args(cmd_args[1:])
self.dbpath = parsed_args.dbpath
self.filterpath = parsed_args.filterpath
self.filtername = parsed_args.filtername
self.filterstring = parsed_args.filterstring
self.savepath = parsed_args.savepath
self.confpath = parsed_args.confpath
self.conf = ConfigLoader(job_conf_path=self.confpath)
def setup(self):
self.data = {}
self.keep_json_all_mapping = True
self.conf = ConfigLoader()
cravat_conf = self.conf.get_cravat_conf()
if 'viewer_effective_digits' in cravat_conf:
self.viewer_effective_digits = cravat_conf[
'viewer_effective_digits']
else:
self.viewer_effective_digits = constants.viewer_effective_digits
def __init__(self, *inargs, **inkwargs):
try:
main_fpath = os.path.abspath(sys.modules[self.__module__].__file__)
self.primary_input_path = None
self.secondary_paths = None
self.output_dir = None
self.output_basename = None
self.plain_output = None
self.job_conf_path = None
self.logger = None
self.dbconn = None
self.cursor = None
self._define_cmd_parser()
self.args = cravat.util.get_args(self.cmd_arg_parser, inargs,
inkwargs)
self.parse_cmd_args(inargs, inkwargs)
if hasattr(self.args, "status_writer") == False:
self.status_writer = None
else:
self.status_writer = self.args.status_writer
if hasattr(self.args, "live") == False:
live = False
else:
live = self.args.live
self.supported_chroms = set(cannonical_chroms)
if live:
return
main_basename = os.path.basename(main_fpath)
if "." in main_basename:
self.module_name = ".".join(main_basename.split(".")[:-1])
else:
self.module_name = main_basename
self.annotator_name = self.module_name
self.module_dir = os.path.dirname(main_fpath)
self.annotator_dir = os.path.dirname(main_fpath)
self.data_dir = os.path.join(self.module_dir, "data")
# Load command line opts
self._setup_logger()
config_loader = ConfigLoader(self.job_conf_path)
self.conf = config_loader.get_module_conf(self.module_name)
self._verify_conf()
self._id_col_name = self.conf["output_columns"][0]["name"]
if "logging_level" in self.conf:
self.logger.setLevel(self.conf["logging_level"].upper())
if "title" in self.conf:
self.annotator_display_name = self.conf["title"]
else:
self.annotator_display_name = os.path.basename(
self.module_dir).upper()
if "version" in self.conf:
self.annotator_version = self.conf["version"]
else:
self.annotator_version = ""
except Exception as e:
self._log_exception(e)
def __init__(self, *inargs, **inkwargs):
try:
main_fpath = os.path.abspath(sys.modules[self.__module__].__file__)
self.primary_input_path = None
self.secondary_paths = None
self.output_dir = None
self.output_basename = None
self.plain_output = None
self.job_conf_path = None
self.logger = None
self.dbconn = None
self.cursor = None
self._define_cmd_parser()
self.args = cravat.util.get_args(self.cmd_arg_parser, inargs,
inkwargs)
self.parse_cmd_args(inargs, inkwargs)
if hasattr(self.args, 'status_writer') == False:
self.status_writer = None
else:
self.status_writer = self.args.status_writer
if hasattr(self.args, 'live') == False:
live = False
else:
live = self.args.live
if live:
return
main_basename = os.path.basename(main_fpath)
if '.' in main_basename:
self.module_name = '.'.join(main_basename.split('.')[:-1])
else:
self.module_name = main_basename
self.annotator_name = self.module_name
self.module_dir = os.path.dirname(main_fpath)
self.annotator_dir = os.path.dirname(main_fpath)
self.data_dir = os.path.join(self.module_dir, 'data')
# Load command line opts
self._setup_logger()
config_loader = ConfigLoader(self.job_conf_path)
self.conf = config_loader.get_module_conf(self.module_name)
self._verify_conf()
self._id_col_name = self.conf['output_columns'][0]['name']
if 'logging_level' in self.conf:
self.logger.setLevel(self.conf['logging_level'].upper())
if 'title' in self.conf:
self.annotator_display_name = self.conf['title']
else:
self.annotator_display_name = os.path.basename(
self.module_dir).upper()
if 'version' in self.conf:
self.annotator_version = self.conf['version']
else:
self.annotator_version = ''
except Exception as e:
self._log_exception(e)
def parse_cmd_args(self, parser, cmd_args):
cmd_args = clean_args(cmd_args)
parsed_args = parser.parse_args(cmd_args)
self.parsed_args = parsed_args
self.dbpath = parsed_args.dbpath
self.filterpath = parsed_args.filterpath
self.filtername = parsed_args.filtername
self.filterstring = parsed_args.filterstring
self.confs = None
if parsed_args.confs is not None:
confs = parsed_args.confs.lstrip('\'').rstrip('\'').replace(
"'", '"')
self.confs = json.loads(confs)
if 'filter' in self.confs:
self.filter = self.confs['filter']
else:
self.filter = None
if parsed_args.output_dir is not None:
self.output_dir = parsed_args.output_dir
else:
self.output_dir = os.path.dirname(self.dbpath)
self.savepath = parsed_args.savepath
if self.savepath is not None and os.path.dirname(self.savepath) == '':
self.savepath = os.path.join(self.output_dir, self.savepath)
self.confpath = parsed_args.confpath
self.conf = ConfigLoader(job_conf_path=self.confpath)
self.module_name = parsed_args.module_name
if self.conf is not None:
self.module_conf = self.conf.get_module_conf(self.module_name)
else:
self.module_conf = None
if hasattr(parsed_args, 'reporttypes'):
self.report_types = parsed_args.reporttypes
self.output_basename = os.path.basename(self.dbpath)[:-7]
status_fname = '{}.status.json'.format(self.output_basename)
self.status_fpath = os.path.join(self.output_dir, status_fname)
self.nogenelevelonvariantlevel = parsed_args.nogenelevelonvariantlevel
if parsed_args.inputfiles is None and parsed_args.dbpath is not None:
db = sqlite3.connect(parsed_args.dbpath)
c = db.cursor()
q = 'select colval from info where colkey="_input_paths"'
c.execute(q)
r = c.fetchone()
if r is not None:
parsed_args.inputfiles = []
s = r[0]
if ' ' in s:
s = s.replace("'", '"')
s = json.loads(r[0].replace("'", '"'))
for k in s:
input_path = s[k]
parsed_args.inputfiles.append(input_path)
self.args = parsed_args
def __init__(self, cmd_args):
# self.module_name = get_caller_name(sys.modules[self.__module__].__file__)
self.module_name = get_caller_name(cmd_args[0])
self.parse_cmd_args(cmd_args)
self._setup_logger()
config_loader = ConfigLoader()
self.conf = config_loader.get_module_conf(self.module_name)
self.fix_col_names()
self.dbconn = None
self.cursor = None
self.cursor_w = None
self._open_db_connection()
self.should_run_annotate = self.check()
def parse_cmd_args(self, cmd_args):
parser = argparse.ArgumentParser()
parser.add_argument('dbpath', help='Path to aggregator output')
parser.add_argument('-f',
dest='filterpath',
default=None,
help='Path to filter file')
parser.add_argument(
'-F',
dest='filtername',
default=None,
help='Name of filter (stored in aggregator output)')
parser.add_argument('--filterstring',
dest='filterstring',
default=None,
help='Filter in JSON')
parser.add_argument('-s',
dest='savepath',
default=None,
help='Path to save file')
parser.add_argument('-c', dest='confpath', help='path to a conf file')
parser.add_argument('-t',
dest='reporttypes',
nargs='+',
default=None,
help='report types')
parser.add_argument('--module-name',
dest='module_name',
default=None,
help='report module name')
parsed_args = parser.parse_args(cmd_args[1:])
self.parsed_args = parsed_args
self.dbpath = parsed_args.dbpath
self.filterpath = parsed_args.filterpath
self.filtername = parsed_args.filtername
self.filterstring = parsed_args.filterstring
self.savepath = parsed_args.savepath
self.confpath = parsed_args.confpath
self.conf = ConfigLoader(job_conf_path=self.confpath)
self.module_name = parsed_args.module_name
if self.conf is not None:
self.module_conf = self.conf.get_module_conf(self.module_name)
else:
self.module_conf = None
self.report_types = parsed_args.reporttypes
self.output_basename = os.path.basename(self.dbpath)[:-7]
self.output_dir = os.path.dirname(self.dbpath)
status_fname = '{}.status.json'.format(self.output_basename)
self.status_fpath = os.path.join(self.output_dir, status_fname)
def __init__(self, cmd_args, status_writer, live=False):
self.live = live
self.t = time.time()
'''
if live:
self.live = live
self.cmd_args = SimpleNamespace()
self.cmd_args.include_sources = []
self.cmd_args.exclude_sources = []
self.input_path = ''
self._setup_logger()
return
'''
self.status_writer = status_writer
main_fpath = cmd_args[0]
main_basename = os.path.basename(main_fpath)
if '.' in main_basename:
self.module_name = '.'.join(main_basename.split('.')[:-1])
else:
self.module_name = main_basename
self.module_dir = os.path.dirname(main_fpath)
self.mapper_dir = os.path.dirname(main_fpath)
self.cmd_parser = None
self.cmd_args = None
self.input_path = None
self.input_dir = None
self.reader = None
self.output_dir = None
self.output_base_fname = None
self.crx_path = None
self.crg_path = None
self.crt_path = None
self.crx_writer = None
self.crg_writer = None
self.crt_writer = None
self.gene_sources = []
#self.primary_gene_source = None
self.gene_info = {}
#self.written_primary_transc = set([])
self._define_main_cmd_args()
self._define_additional_cmd_args()
self._parse_cmd_args(cmd_args)
self._setup_logger()
config_loader = ConfigLoader()
self.conf = config_loader.get_module_conf(self.module_name)
self.cravat_version = pkg_resources.get_distribution(
'open-cravat').version
def __init__(self, cmd_args):
try:
self.logger = None
main_fpath = os.path.abspath(sys.modules[self.__module__].__file__)
main_basename = os.path.basename(main_fpath)
if "." in main_basename:
self.annotator_name = ".".join(main_basename.split(".")[:-1])
else:
self.annotator_name = main_basename
self.annotator_dir = os.path.dirname(main_fpath)
self.data_dir = os.path.join(self.annotator_dir, "data")
# Load command line opts
self.primary_input_path = None
self.secondary_paths = None
self.output_dir = None
self.output_basename = None
self.plain_output = None
self.job_conf_path = None
self.parse_cmd_args(cmd_args)
# Make output dir if it doesn't exist
if not (os.path.exists(self.output_dir)):
os.makedirs(self.output_dir)
self._setup_logger()
config_loader = ConfigLoader(self.job_conf_path)
self.conf = config_loader.get_module_conf(self.annotator_name)
self._verify_conf()
self._id_col_name = self.conf["output_columns"][0]["name"]
if "logging_level" in self.conf:
self.logger.setLevel(self.conf["logging_level"].upper())
if "title" in self.conf:
self.annotator_display_name = self.conf["title"]
else:
self.annotator_display_name = os.path.basename(
self.annotator_dir
).upper()
if "version" in self.conf:
self.annotator_version = self.conf["version"]
self.logger.info("Initialized %s" % self.annotator_name)
self.dbconn = None
self.cursor = None
except Exception as e:
self._log_exception(e)
def __init__(self, cmd_args, status_writer):
try:
self.status_writer = status_writer
self.logger = None
main_fpath = cmd_args[0]
main_basename = os.path.basename(main_fpath)
if '.' in main_basename:
self.annotator_name = '.'.join(main_basename.split('.')[:-1])
else:
self.annotator_name = main_basename
self.annotator_dir = os.path.dirname(main_fpath)
self.data_dir = os.path.join(self.annotator_dir, 'data')
# Load command line opts
self.primary_input_path = None
self.secondary_paths = None
self.output_dir = None
self.output_basename = None
self.plain_output = None
self.job_conf_path = None
self.parse_cmd_args(cmd_args)
# Make output dir if it doesn't exist
if not (os.path.exists(self.output_dir)):
os.makedirs(self.output_dir)
self._setup_logger()
config_loader = ConfigLoader(self.job_conf_path)
self.conf = config_loader.get_module_conf(self.annotator_name)
self._verify_conf()
self._id_col_name = self.conf['output_columns'][0]['name']
if 'logging_level' in self.conf:
self.logger.setLevel(self.conf['logging_level'].upper())
if 'title' in self.conf:
self.annotator_display_name = self.conf['title']
else:
self.annotator_display_name = os.path.basename(
self.annotator_dir).upper()
if 'version' in self.conf:
self.annotator_version = self.conf['version']
else:
self.annotator_version = ''
self.dbconn = None
self.cursor = None
except Exception as e:
self._log_exception(e)
def __init__(self, *inargs, **inkwargs):
self.cmd_parser = None
self.input_path = None
self.input_dir = None
self.reader = None
self.output_dir = None
self.output_base_fname = None
self.crx_path = None
self.crg_path = None
self.crt_path = None
self.crx_writer = None
self.crg_writer = None
self.crt_writer = None
self._define_main_cmd_args()
self._define_additional_cmd_args()
self._parse_cmd_args(inargs, inkwargs)
if hasattr(self.args, "status_writer") == False:
status_writer = None
else:
status_writer = self.args.status_writer
if hasattr(self.args, "live") == False:
live = False
else:
live = self.args.live
self.live = live
self.t = time.time()
self.status_writer = status_writer
main_fpath = self.args.script_path
main_basename = os.path.basename(main_fpath)
if "." in main_basename:
self.module_name = ".".join(main_basename.split(".")[:-1])
else:
self.module_name = main_basename
self.module_dir = os.path.dirname(main_fpath)
self.mapper_dir = os.path.dirname(main_fpath)
self.gene_sources = []
# self.primary_gene_source = None
self.gene_info = {}
# self.written_primary_transc = set([])
self._setup_logger()
config_loader = ConfigLoader()
self.conf = config_loader.get_module_conf(self.module_name)
self.cravat_version = pkg_resources.get_distribution("open-cravat").version
def parse_cmd_args(self, parser, cmd_args):
cmd_args = clean_args(cmd_args)
parsed_args = parser.parse_args(cmd_args)
self.parsed_args = parsed_args
self.dbpath = parsed_args.dbpath
self.filterpath = parsed_args.filterpath
self.filtername = parsed_args.filtername
self.filterstring = parsed_args.filterstring
self.confs = None
if parsed_args.confs is not None:
confs = parsed_args.confs.lstrip('\'').rstrip('\'').replace(
"'", '"')
self.confs = json.loads(confs)
if 'filter' in self.confs:
self.filter = self.confs['filter']
else:
self.filter = None
if parsed_args.output_dir is not None:
self.output_dir = parsed_args.output_dir
else:
self.output_dir = os.path.dirname(self.dbpath)
self.savepath = parsed_args.savepath
if self.savepath is not None and os.path.dirname(self.savepath) == '':
self.savepath = os.path.join(self.output_dir, self.savepath)
self.confpath = parsed_args.confpath
self.conf = ConfigLoader(job_conf_path=self.confpath)
self.module_name = parsed_args.module_name
if self.conf is not None:
self.module_conf = self.conf.get_module_conf(self.module_name)
else:
self.module_conf = None
if hasattr(parsed_args, 'reporttypes'):
self.report_types = parsed_args.reporttypes
self.output_basename = os.path.basename(self.dbpath)[:-7]
status_fname = '{}.status.json'.format(self.output_basename)
self.status_fpath = os.path.join(self.output_dir, status_fname)
self.nogenelevelonvariantlevel = parsed_args.nogenelevelonvariantlevel
self.args = parsed_args
def parse_cmd_args(self, inargs, inkwargs):
parsed_args = cravat.util.get_args(parser, inargs, inkwargs)
self.parsed_args = parsed_args
if parsed_args.md is not None:
constants.custom_modules_dir = parsed_args.md
self.dbpath = parsed_args.dbpath
self.filterpath = parsed_args.filterpath
self.filtername = parsed_args.filtername
self.filterstring = parsed_args.filterstring
self.filtersql = parsed_args.filtersql
self.filter = parsed_args.filter
self.confs = {}
if parsed_args.output_dir is not None:
self.output_dir = parsed_args.output_dir
else:
self.output_dir = os.path.dirname(self.dbpath)
self.savepath = parsed_args.savepath
if self.savepath is not None and os.path.dirname(self.savepath) == "":
self.savepath = os.path.join(self.output_dir, self.savepath)
self.confpath = parsed_args.confpath
self.conf = ConfigLoader(job_conf_path=self.confpath)
self.module_name = parsed_args.module_name
if self.module_name in self.conf._all:
self.confs.update(self.conf._all[self.module_name])
if self.conf is not None:
self.module_conf = self.conf.get_module_conf(self.module_name)
else:
self.module_conf = None
if hasattr(parsed_args, "reporttypes"):
self.report_types = parsed_args.reporttypes
if hasattr(parsed_args, "conf") and parsed_args.conf is not None:
self.confs.update(parsed_args.conf)
if parsed_args.confs is not None:
confs = parsed_args.confs.lstrip("'").rstrip("'").replace("'", '"')
if self.confs is None:
self.confs = json.loads(confs)
else:
self.confs.update(json.loads(confs))
# Chooses filter.
if self.filter is None:
if self.confs is not None and "filter" in self.confs:
self.filter = self.confs["filter"]
local = au.mic.get_local()
if (self.filter is None and self.filterpath is None
and self.filtername is None and self.filterstring is None
and parsed_args.package is not None
and parsed_args.package in local
and "filter" in local[parsed_args.package].conf):
self.filter = local[parsed_args.package].conf["filter"]
self.output_basename = os.path.basename(self.dbpath)[:-7]
status_fname = "{}.status.json".format(self.output_basename)
self.status_fpath = os.path.join(self.output_dir, status_fname)
self.nogenelevelonvariantlevel = parsed_args.nogenelevelonvariantlevel
if parsed_args.inputfiles is None and parsed_args.dbpath is not None:
db = sqlite3.connect(parsed_args.dbpath)
c = db.cursor()
q = 'select colval from info where colkey="_input_paths"'
c.execute(q)
r = c.fetchone()
if r is not None:
parsed_args.inputfiles = []
s = r[0]
if " " in s:
s = s.replace("'", '"')
s = s.replace("\\", "\\\\\\\\")
s = json.loads(s)
for k in s:
input_path = s[k]
parsed_args.inputfiles.append(input_path)
c.close()
db.close()
if hasattr(parsed_args, "status_writer"):
self.status_writer = parsed_args.status_writer
else:
self.status_writer = None
self.concise_report = parsed_args.concise_report
self.extract_columns_multilevel = self.get_standardized_module_option(
self.confs.get("extract-columns", {}))
self.args = parsed_args
import glob
import platform
import signal
import multiprocessing as mp
import asyncio
import importlib
from multiprocessing import Process, Pipe, Value, Manager, Queue
from queue import Empty
from cravat import constants
from cravat import get_live_annotator, get_live_mapper
import signal
import gzip
from cravat.cravat_util import max_version_supported_for_migration
import cravat.util
cfl = ConfigLoader()
class FileRouter(object):
def __init__(self):
self.root = os.path.dirname(__file__)
self.input_fname = 'input'
self.report_extensions = {
'text': '.tsv',
'excel': '.xlsx',
'vcf': '.vcf'
}
self.db_extension = '.sqlite'
self.log_extension = '.log'
self.status_extension = '.status.json'
self.job_statuses = {}
def setup (self):
self.data = {}
self.keep_json_all_mapping = True
self.conf = ConfigLoader()
def get_cravat_conf ():
from cravat.config_loader import ConfigLoader
confpath = get_main_conf_path()
conf = ConfigLoader()
cravat_conf = conf.get_cravat_conf()
return cravat_conf
def __init__(self, dir_path, name=None):
self.directory = dir_path
if name is None:
self.name = os.path.basename(self.directory)
else:
self.name = name
self.script_path = os.path.join(self.directory, self.name+'.py')
#if importlib.util.find_spec('cython') is not None:
# pyx_path = self.script_path + 'x'
# if os.path.exists(pyx_path):
# self.script_path = pyx_path
self.script_exists = os.path.exists(self.script_path)
self.conf_path = os.path.join(self.directory, self.name+'.yml')
self.conf_exists = os.path.exists(self.conf_path)
self.exists = self.conf_exists
startofinstall_path = os.path.join(self.directory, 'startofinstall')
if os.path.exists(startofinstall_path):
endofinstall_path = os.path.join(self.directory, 'endofinstall')
if os.path.exists(endofinstall_path):
self.exists = True
else:
self.exists = False
self.data_dir = os.path.join(dir_path, 'data')
self.data_dir_exists = os.path.isdir(self.data_dir)
self.has_data = self.data_dir_exists \
and len(os.listdir(self.data_dir)) > 0
self.test_dir = os.path.join(dir_path, 'test')
self.test_dir_exists = os.path.isdir(self.test_dir)
self.has_test = self.test_dir_exists \
and os.path.isfile(os.path.join(self.test_dir, 'input')) \
and os.path.isfile(os.path.join(self.test_dir, 'key'))
self.readme_path = os.path.join(self.directory, self.name+'.md')
self.readme_exists = os.path.exists(self.readme_path)
if self.readme_exists:
with open(self.readme_path) as f:
self.readme = f.read()
else:
self.readme = ''
self.helphtml_path = os.path.join(self.directory, 'help.html')
self.helphtml_exists = os.path.exists(self.helphtml_path)
self.conf = {}
if self.conf_exists:
from cravat.config_loader import ConfigLoader
conf = ConfigLoader()
self.conf = conf.get_module_conf(self.name)
self.type = self.conf.get('type')
self.version = self.conf.get('version')
self.description = self.conf.get('description')
self.hidden = self.conf.get('hidden',False)
dev_dict = self.conf.get('developer')
if not(type(dev_dict)==dict):
dev_dict = {}
self.developer = get_developer_dict(**dev_dict)
if 'type' not in self.conf:
self.conf['type'] = 'unknown'
self.type = self.conf['type']
self.level = self.conf.get('level')
self.input_format = self.conf.get('input_format')
self.secondary_module_names = list(self.conf.get('secondary_inputs',{}))
if self.type == 'annotator':
if self.level == 'variant':
self.output_suffix = self.name + '.var'
elif self.level == 'gene':
self.output_suffix = self.name + '.gen'
else:
self.output_suffix = self. name + '.' + self.type
self.title = self.conf.get('title',self.name)
self.disk_size = None
self.tags = self.conf.get('tags',[])
self.datasource = str(self.conf.get('datasource',''))
self.smartfilters = self.conf.get('smartfilters')
self.groups = self.conf.get('groups')