def panel(panel, version, update_date, update_version):
"""
Update a panel in the database
"""
adapter = store
# Check that the panel exists
panel_obj = adapter.gene_panel(panel, version=version)
if not panel_obj:
LOG.warning("Panel %s (version %s) could not be found" % (panel, version))
click.Abort()
date_obj = None
if update_date:
try:
date_obj = get_date(update_date)
except Exception as err:
LOG.warning(err)
click.Abort()
update_panel(
adapter,
panel,
panel_version=panel_obj["version"],
new_version=update_version,
new_date=date_obj,
)
def panel(context, panel, version, update_date, update_version):
"""
Update a panel in the database
"""
adapter = context.obj['adapter']
#Check that the panel exists
panel_obj = adapter.gene_panel(panel, version=version)
# for panel_obj in adapter.panel_collection.find({'panel_name': 'CAN'}):
# print(panel_obj['panel_name'], panel_obj['version'])
#
# print(panel_obj)
if not panel_obj:
LOG.warning("Panel %s (version %s) could not be found" % (panel, version))
context.abort()
date_obj = None
if update_date:
try:
date_obj = get_date(update_date)
except Exception as err:
LOG.warning(err)
context.abort()
updated_panel = update_panel(
adapter,
panel,
panel_version=panel_obj['version'],
new_version=update_version,
new_date=date_obj
)
def analysis_date_to_Datetime(cls, analysis_date):
"""Check if analysis_date is on datetime format or string convertible to datetime.
Otherwise return now."""
if isinstance(analysis_date, datetime.datetime):
return analysis_date
try:
return get_date(analysis_date)
except ValueError as err:
LOG.warning("Analysis date is on wrong format: {}".format(err))
LOG.warning("Setting analysis date to todays date")
return datetime.datetime.now()
def panel(context, date, name, version, panel_type, panel_id, path, institute):
"""Add a gene panel to the database."""
adapter = context.obj['adapter']
f = get_file_handle(path)
for line in f:
line = line.rstrip()
if line.startswith('##'):
info = line[2:].split('=')
name = info[0]
value = info[1]
if name == 'panel_id':
panel_id = value
elif name == 'institute':
institute = value
elif name == 'version':
version = float(value)
elif name == 'date':
date = value
elif name == 'display_name':
name = value
if version:
existing_panel = adapter.gene_panel(panel_id, version)
else:
existing_panel = adapter.gene_panel(panel_id)
if existing_panel:
log.debug("found existing panel")
name = name or existing_panel.display_name or panel_id
institute = institute or existing_panel.institute
try:
date = get_date(date)
except ValueError as error:
log.warning(error)
context.abort()
info = {
'file': path,
'institute': institute,
'type': panel_type,
'date': date,
'version': version,
'panel_name': panel_id,
'full_name': name,
}
load_panel(adapter=adapter, panel_info=info)
def test_update_panel_date(adapter, case_obj, dummypanel_obj):
adapter.case_collection.insert_one(case_obj)
adapter.panel_collection.insert_one(dummypanel_obj)
## GIVEN an adapter with a case with gene panels
new_date_obj = get_date('2015-03-12')
case_obj = adapter.case_collection.find_one()
panel_obj = adapter.panel_collection.find_one()
case_id = case_obj['_id']
# There is infirmation about a panel both in the panel collection
# and on the case object. This is fine until one starts to manipulate the objects
case_panel = case_obj['panels'][0]
panel_version = case_panel['version']
panel_name = case_panel['panel_name']
panel_id = case_panel['panel_id']
## WHEN updating the panel version
update_panel(adapter,
panel_name,
panel_version=None,
new_date=new_date_obj)
## THEN assert that the panel version was updated both in panel and case
updated_panel_obj = adapter.panel_collection.find_one({
'panel_name':
panel_name,
'version':
panel_version
})
assert updated_panel_obj['date'] == new_date_obj
case_obj = adapter.case_collection.find_one({'_id': case_id})
for panel in case_obj['panels']:
assert panel['updated_at'] == new_date_obj
def parse_panel_app_panel(panel_info,
hgnc_map,
institute="cust000",
panel_type="clinical"):
"""Parse a PanelApp panel
Args:
panel_info(dict)
hgnc_map(dict): Map from symbol to hgnc ids
institute(str)
panel_type(str)
Returns:
gene_panel(dict)
"""
date_format = "%Y-%m-%dT%H:%M:%S.%f"
gene_panel = {}
gene_panel["version"] = float(panel_info["version"])
gene_panel["date"] = get_date(panel_info["Created"][:-1],
date_format=date_format)
gene_panel["display_name"] = panel_info["SpecificDiseaseName"]
gene_panel["institute"] = institute
gene_panel["panel_type"] = panel_type
LOG.info("Parsing panel %s", gene_panel["display_name"])
gene_panel["genes"] = []
nr_low_confidence = 1
nr_genes = 0
for nr_genes, gene in enumerate(panel_info["Genes"], 1):
gene_info = parse_panel_app_gene(gene, hgnc_map)
if not gene_info:
nr_low_confidence += 1
continue
gene_panel["genes"].append(gene_info)
LOG.info("Number of genes in panel %s", nr_genes)
LOG.info("Number of low confidence genes in panel %s", nr_low_confidence)
return gene_panel
def test_update_panel_date(adapter, case_obj, testpanel_obj):
adapter.case_collection.insert_one(case_obj)
adapter.panel_collection.insert_one(testpanel_obj)
## GIVEN an adapter with a case with gene panels
new_date_obj = get_date("2015-03-12")
case_obj = adapter.case_collection.find_one()
panel_obj = adapter.panel_collection.find_one()
case_id = case_obj["_id"]
# There is infirmation about a panel both in the panel collection
# and on the case object. This is fine until one starts to manipulate the objects
case_panel = case_obj["panels"][0]
panel_version = case_panel["version"]
panel_name = case_panel["panel_name"]
panel_id = case_panel["panel_id"]
## WHEN updating the panel version
update_panel(adapter,
panel_name,
panel_version=None,
new_date=new_date_obj)
## THEN assert that the panel version was updated both in panel and case
updated_panel_obj = adapter.panel_collection.find_one({
"panel_name":
panel_name,
"version":
panel_version
})
assert updated_panel_obj["date"] == new_date_obj
case_obj = adapter.case_collection.find_one({"_id": case_id})
for panel in case_obj["panels"]:
assert panel["updated_at"] == new_date_obj
def get_panel_info(panel_lines=None,
panel_id=None,
institute=None,
version=None,
date=None,
display_name=None):
"""Parse metadata for a gene panel
For historical reasons it is possible to include all information about a gene panel in the
header of a panel file. This function parses the header.
Args:
panel_lines(iterable(str))
Returns:
panel_info(dict): Dictionary with panel information
"""
panel_info = {
'panel_id': panel_id,
'institute': institute,
'version': version,
'date': date,
'display_name': display_name,
}
if panel_lines:
for line in panel_lines:
line = line.rstrip()
if not line.startswith('##'):
break
info = line[2:].split('=')
field = info[0]
value = info[1]
if not panel_info.get(field):
panel_info[field] = value
panel_info['date'] = get_date(panel_info['date'])
return panel_info
def get_panel_info(panel_lines=None, panel_id=None, institute=None, **kwargs):
"""Parse metadata for a gene panel
For historical reasons it is possible to include all information about a gene panel in the
header of a panel file. This function parses the header.
Args:
panel_lines(iterable(str))
panel_id(str)
institute(str)
maintainer(list(user._id))
Returns:
panel_info(dict): Dictionary with panel information
"""
panel_info = {
"panel_id": panel_id,
"institute": institute,
"version": kwargs.get("version"),
"maintainer": kwargs.get("maintainer"),
"date": kwargs.get("date"),
"display_name": kwargs.get("display_name"),
}
if panel_lines:
for line in panel_lines:
line = line.rstrip()
if not line.startswith("##"):
break
info = line[2:].split("=")
field = info[0]
value = info[1]
if not panel_info.get(field):
panel_info[field] = value
panel_info["date"] = get_date(panel_info["date"])
return panel_info
def test_update_panel_date(panel_database, case_obj):
adapter = panel_database
adapter._add_case(case_obj)
## GIVEN an adapter with a case with gene panels
new_date_obj = get_date('2015-03-12')
case_obj = adapter.case_collection.find_one()
panel_obj = adapter.panel_collection.find_one()
case_id = case_obj['_id']
# There is infirmation about a panel both in the panel collection
# and on the case object. This is fine until one starts to manipulate the objects
case_panel = case_obj['panels'][0]
panel_version = case_panel['version']
panel_name = case_panel['panel_name']
panel_id = case_panel['panel_id']
## WHEN updating the panel version
update_panel(adapter, panel_name, panel_version=None, new_date=new_date_obj)
## THEN assert that the panel version was updated both in panel and case
updated_panel_obj = adapter.panel_collection.find_one(
{
'panel_name': panel_name,
'version': panel_version
})
assert updated_panel_obj['date'] == new_date_obj
case_obj = adapter.case_collection.find_one({'_id': case_id})
for panel in case_obj['panels']:
assert panel['updated_at'] == new_date_obj
def get_correct_date(date_info):
"""Convert dateinfo to correct date
Args:
dateinfo: Something that represents a date
Returns:
correct_date(datetime.datetime)
"""
if isinstance(date_info, datetime.datetime):
return date_info
if isinstance(date_info, str):
try:
correct_date = get_date(date_info)
except ValueError as err:
LOG.warning("Analysis date is on wrong format")
LOG.info("Setting analysis date to todays date")
correct_date = datetime.datetime.now()
return correct_date
LOG.info("Setting analysis date to todays date")
return datetime.datetime.now()
def test_invalid_date():
date_str = "20150510"
with pytest.raises(ValueError):
date_obj = get_date(date_str)
def load_omim_panel(self, api_key, institute=None):
"""Create and load the OMIM-AUTO panel"""
existing_panel = self.gene_panel(panel_id='OMIM-AUTO')
if not existing_panel:
LOG.warning("OMIM-AUTO does not exists in database")
LOG.info('Creating a first version')
version = 1.0
if existing_panel:
version = float(math.floor(existing_panel['version']) + 1)
LOG.info("Setting version to %s", version)
try:
mim_files = fetch_mim_files(api_key=api_key, genemap2=True, mim2genes=True)
except Exception as err:
raise err
date_string = None
# Get the correct date when omim files where released
for line in mim_files['genemap2']:
if 'Generated' in line:
date_string = line.split(':')[-1].lstrip().rstrip()
date_obj = get_date(date_string)
if existing_panel:
if existing_panel['date'] == date_obj:
LOG.warning("There is no new version of OMIM")
return
panel_data = {}
panel_data['path'] = None
panel_data['type'] = 'clinical'
panel_data['date'] = date_obj
panel_data['panel_id'] = 'OMIM-AUTO'
panel_data['institute'] = institute or 'cust002'
panel_data['version'] = version
panel_data['display_name'] = 'OMIM-AUTO'
panel_data['genes'] = []
alias_genes = self.genes_by_alias()
genes = get_omim_panel_genes(
genemap2_lines = mim_files['genemap2'],
mim2gene_lines = mim_files['mim2genes'],
alias_genes = alias_genes,
)
for gene in genes:
panel_data['genes'].append(gene)
panel_obj = build_panel(panel_data, self)
if existing_panel:
new_genes = self.compare_mim_panels(existing_panel, panel_obj)
if new_genes:
self.update_mim_version(new_genes, panel_obj, old_version=existing_panel['version'])
else:
LOG.info("The new version of omim does not differ from the old one")
LOG.info("No update is added")
return
self.add_gene_panel(panel_obj)
def test_valid_date():
date_str = "2015-05-10"
date_obj = get_date(date_str)
assert isinstance(date_obj, datetime.datetime)
def test_valid_date_no_date():
date_str = None
date_obj = get_date(date_str)
assert isinstance(date_obj, datetime.datetime)
def test_invalid_date():
date_str = "20150510"
with pytest.raises(ValueError):
date_obj = get_date(date_str)
def test_valid_date():
date_str = "2015-05-10"
date_obj = get_date(date_str)
assert isinstance(date_obj, datetime.datetime)
def test_valid_date_no_date():
date_str = None
date_obj = get_date(date_str)
assert isinstance(date_obj, datetime.datetime)
def panel(
panel, version, update_date, update_version, add_maintainer, revoke_maintainer
):
"""
Update a panel in the database
"""
adapter = store
# Check that the panel exists
panel_obj = adapter.gene_panel(panel, version=version)
if not panel_obj:
LOG.warning("Panel %s (version %s) could not be found", panel, version)
raise click.Abort()
date_obj = None
if update_date:
try:
date_obj = get_date(update_date)
except Exception as err:
LOG.warning(err)
raise click.Abort()
# Any mintainer updates?
new_maintainer = None
if add_maintainer:
user_obj = adapter.user(user_id=add_maintainer)
if not user_obj:
# Check if maintainers exist in the user database
LOG.warning(
"Maintainer user id %s does not exist in user database", add_maintainer
)
raise click.Abort()
new_maintainer = panel_obj.get("maintainer", [])
if add_maintainer in new_maintainer:
LOG.warning("User %s already in maintainer list.", add_maintainer)
raise click.Abort()
new_maintainer.append(add_maintainer)
if revoke_maintainer:
current_maintainers = panel_obj.get("maintainer", [])
try:
current_maintainers.remove(revoke_maintainer)
new_maintainer = current_maintainers
except ValueError:
LOG.warning(
"Maintainer user id %s is not a maintainer for panel %s. Current maintainers: %s",
revoke_maintainer,
panel,
current_maintainers,
)
update_panel(
adapter,
panel,
panel_version=panel_obj["version"],
new_version=update_version,
new_maintainer=new_maintainer,
new_date=date_obj,
)
def load_omim_panel(self, genemap2_lines, mim2gene_lines, institute=None):
"""Create and load the OMIM-AUTO panel
If the panel already exists, update with new information and increase version
Args:
genemap_lines(iterable(str)): The genemap2 file information
mim2gene_lines(iterable(str)): The mim2genes file information
institute(str): What institute that is responsible. Default: 'cust002'
"""
institute = institute or "cust002"
existing_panel = self.gene_panel(panel_id="OMIM-AUTO")
if not existing_panel:
LOG.warning("OMIM-AUTO does not exists in database")
LOG.info("Creating a first version")
version = 1.0
if existing_panel:
version = float(math.floor(existing_panel["version"]) + 1)
LOG.info("Setting version to %s", version)
date_string = None
# Get the correct date when omim files where released
for line in genemap2_lines:
if "Generated" in line:
date_string = line.split(":")[-1].strip()
break
date_obj = get_date(date_string)
if existing_panel:
if existing_panel["date"] == date_obj:
LOG.warning("There is no new version of OMIM")
return
panel_data = {
"path": None,
"type": "clinical",
"date": date_obj,
"panel_id": "OMIM-AUTO",
"institute": institute,
"version": version,
"display_name": "OMIM-AUTO",
"genes": [],
}
alias_genes = self.genes_by_alias()
genes = get_omim_panel_genes(
genemap2_lines=genemap2_lines,
mim2gene_lines=mim2gene_lines,
alias_genes=alias_genes,
)
for gene in genes:
panel_data["genes"].append(gene)
panel_obj = build_panel(panel_data, self)
if existing_panel:
new_genes = self.compare_mim_panels(existing_panel, panel_obj)
if not new_genes:
LOG.info("The new version of omim does not differ from the old one")
LOG.info("No update is added")
return
self.update_mim_version(
new_genes, panel_obj, old_version=existing_panel["version"]
)
self.add_gene_panel(panel_obj)
def load_omim_panel(self, api_key, institute=None):
"""Create and load the OMIM-AUTO panel"""
existing_panel = self.gene_panel(panel_id='OMIM-AUTO')
if not existing_panel:
LOG.warning("OMIM-AUTO does not exists in database")
LOG.info('Creating a first version')
version = 1.0
if existing_panel:
version = float(math.floor(existing_panel['version']) + 1)
LOG.info("Setting version to %s", version)
try:
mim_files = fetch_mim_files(api_key=api_key, genemap2=True, mim2genes=True)
except Exception as err:
raise err
date_string = None
# Get the correct date when omim files where released
for line in mim_files['genemap2']:
if 'Generated' in line:
date_string = line.split(':')[-1].lstrip().rstrip()
date_obj = get_date(date_string)
if existing_panel:
if existing_panel['date'] == date_obj:
LOG.warning("There is no new version of OMIM")
return
panel_data = {}
panel_data['path'] = None
panel_data['type'] = 'clinical'
panel_data['date'] = date_obj
panel_data['panel_id'] = 'OMIM-AUTO'
panel_data['institute'] = institute or 'cust002'
panel_data['version'] = version
panel_data['display_name'] = 'OMIM-AUTO'
panel_data['genes'] = []
alias_genes = self.genes_by_alias()
genes = get_omim_panel_genes(
genemap2_lines = mim_files['genemap2'],
mim2gene_lines = mim_files['mim2genes'],
alias_genes = alias_genes,
)
for gene in genes:
panel_data['genes'].append(gene)
panel_obj = build_panel(panel_data, self)
if existing_panel:
new_genes = self.compare_mim_panels(existing_panel, panel_obj)
if new_genes:
self.update_mim_version(new_genes, panel_obj, old_version=existing_panel['version'])
else:
LOG.info("The new version of omim does not differ from the old one")
LOG.info("No update is added")
return
self.add_gene_panel(panel_obj)
