def delete_individuals(project, individual_guids, user):
"""Delete one or more individuals
Args:
project (object): Django ORM model for project
individual_guids (list): GUIDs of individuals to delete
Returns:
list: Family objects for families with deleted individuals
"""
individuals_to_delete = Individual.objects.filter(
family__project=project, guid__in=individual_guids)
Sample.bulk_delete(user, individual__family__project=project, individual__guid__in=individual_guids)
IgvSample.bulk_delete(user, individual__family__project=project, individual__guid__in=individual_guids)
families = {individual.family for individual in individuals_to_delete}
Individual.bulk_delete(user, queryset=individuals_to_delete)
update_pedigree_images(families, user)
families_with_deleted_individuals = list(families)
return families_with_deleted_individuals
def delete_individuals(project, individual_guids):
"""Delete one or more individuals
Args:
project (object): Django ORM model for project
individual_guids (list): GUIDs of individuals to delete
Returns:
list: Family objects for families with deleted individuals
"""
individuals_to_delete = Individual.objects.filter(
family__project=project, guid__in=individual_guids)
samples_to_delete = Sample.objects.filter(
individual__family__project=project,
individual__guid__in=individual_guids)
for sample in samples_to_delete:
logger.info("Deleting sample: %s" % sample)
sample.delete()
families = {}
for individual in individuals_to_delete:
families[individual.family.family_id] = individual.family
# delete phenotips records
try:
delete_phenotips_patient(project, individual)
except (PhenotipsException, ValueError) as e:
logger.error(
"Error: couldn't delete patient from phenotips: {} {} ({})".
format(individual.phenotips_eid, individual, e))
# delete Individual
delete_seqr_model(individual)
update_pedigree_images(families.values())
families_with_deleted_individuals = list(families.values())
return families_with_deleted_individuals
def test_update_pedigree_images(self, mock_tempfile, mock_run,
mock_randint, mock_logger):
mock_tempfile.gettempdir.return_value = '/tmp'
mock_tempfile_file = mock_tempfile.NamedTemporaryFile.return_value.__enter__.return_value
mock_tempfile_file.name = 'temp.fam'
mock_randint.return_value = 123456
def _mock_paint(command, **kwargs):
with open(command[-1], 'wb') as f:
f.write(b'\xff\xd8\xff')
return MOCK_PAINT_PROCESS
mock_run.side_effect = _mock_paint
test_families = Family.objects.filter(guid='F000001_1')
update_pedigree_images(test_families, None)
pedigree_image = test_families.first().pedigree_image
self.assertTrue(bool(pedigree_image))
self.assertEqual(pedigree_image.name,
'pedigree_images/pedigree_image_123456.png')
os.remove(pedigree_image.path)
mock_run.assert_called_with([
'perl', '/seqr/management/commands/HaploPainter1.043.pl', '-b',
'-outformat', 'png', '-pedfile', 'temp.fam', '-family', '1',
'-outfile', '/tmp/pedigree_image_123456.png'
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True)
mock_tempfile_file.write.assert_has_calls([
mock.call('\t'.join(
['1', 'NA19675_1', 'NA19678', 'NA19679', '1', '2'])),
mock.call('\n'),
mock.call('\t'.join(['1', 'NA19678', '0', '0', '1', '1'])),
mock.call('\n'),
mock.call('\t'.join(['1', 'NA19679', '0', '0', '2', '1'])),
mock.call('\n'),
])
mock_logger.info.assert_any_call('Generated pedigree', None)
mock_logger.warning.assert_not_called()
mock_logger.error.assert_not_called()
# Create placeholder when only has one parent
Sample.objects.get(guid='S000130_na19678').delete()
Individual.objects.get(individual_id='NA19678').delete()
mock_tempfile_file.write.reset_mock()
mock_run.reset_mock()
mock_logger.reset_mock()
MOCK_PAINT_PROCESS.returncode = 1
MOCK_PAINT_PROCESS.stdout = 'Error!'
update_pedigree_images(test_families, None)
pedigree_image = test_families.first().pedigree_image
self.assertTrue(bool(pedigree_image))
self.assertEqual(pedigree_image.name,
'pedigree_images/pedigree_image_123456.png')
os.remove(pedigree_image.path)
mock_run.assert_called_with([
'perl', '/seqr/management/commands/HaploPainter1.043.pl', '-b',
'-outformat', 'png', '-pedfile', 'temp.fam', '-family', '1',
'-outfile', '/tmp/pedigree_image_123456.png'
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True)
mock_tempfile_file.write.assert_has_calls([
mock.call('\t'.join(
['1', 'NA19675_1', 'placeholder_123456', 'NA19679', '1',
'2'])),
mock.call('\n'),
mock.call('\t'.join(['1', 'NA19679', '0', '0', '2', '1'])),
mock.call('\n'),
mock.call('\t'.join(
['1', 'placeholder_123456', '0', '0', '1', '9'])),
mock.call('\n'),
])
mock_logger.warning.assert_not_called()
mock_logger.error.assert_called_with(
'Generated pedigree image for family 1 with exit status 1: Error!',
None)
# Do not generate for families with one individual
mock_tempfile_file.write.reset_mock()
mock_run.reset_mock()
mock_logger.reset_mock()
one_individual_families = Family.objects.filter(guid='F000003_3')
update_pedigree_images(one_individual_families, None)
pedigree_image = one_individual_families.first().pedigree_image
self.assertFalse(bool(pedigree_image))
mock_run.assert_not_called()
mock_tempfile_file.write.assert_not_called()
mock_logger.warning.assert_not_called()
mock_logger.error.assert_not_called()
# Do not generate for families with no parental inheritance specified
two_individual_families = Family.objects.filter(guid='F000012_12')
update_pedigree_images(two_individual_families, None)
pedigree_image = two_individual_families.first().pedigree_image
self.assertFalse(bool(pedigree_image))
mock_run.assert_not_called()
mock_tempfile_file.write.assert_not_called()
mock_logger.warning.assert_called_with(
'Unable to generate for pedigree image for family 12: no parents specified',
None)
mock_logger.error.assert_not_called()
# Alert when generation fails
mock_logger.reset_mock()
mock_run.side_effect = lambda *args, **kwargs: MOCK_PAINT_PROCESS
update_pedigree_images(test_families, None)
pedigree_image = test_families.first().pedigree_image
self.assertFalse(bool(pedigree_image))
mock_run.assert_called_with([
'perl', '/seqr/management/commands/HaploPainter1.043.pl', '-b',
'-outformat', 'png', '-pedfile', 'temp.fam', '-family', '1',
'-outfile', '/tmp/pedigree_image_123456.png'
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True)
records = [[
'1', 'NA19675_1', 'placeholder_123456', 'NA19679', '1', '2'
], ['1', 'NA19679', '0', '0', '2', '1'],
['1', 'placeholder_123456', '0', '0', '1', '9']]
mock_tempfile_file.write.assert_has_calls([
mock.call('\t'.join(records[0])),
mock.call('\n'),
mock.call('\t'.join(records[1])),
mock.call('\n'),
mock.call('\t'.join(records[2])),
mock.call('\n'),
])
mock_logger.warning.assert_not_called()
mock_logger.error.assert_called_with(
'Failed to generate pedigree image for family 1: Error!',
None,
detail={'ped_file': records})
def add_or_update_individuals_and_families(project, individual_records, user):
"""
Add or update individual and family records in the given project.
Args:
project (object): Django ORM model for the project to add families to
individual_records (list): A list of JSON records representing individuals. See
the return value of pedigree_info_utils#convert_fam_file_rows_to_json(..)
Return:
2-tuple: updated_families, updated_individuals containing Django ORM models
"""
updated_families = set()
updated_individuals = set()
parent_updates = []
family_ids = {_get_record_family_id(record) for record in individual_records}
families_by_id = {f.family_id: f for f in Family.objects.filter(project=project, family_id__in=family_ids)}
missing_family_ids = family_ids - set(families_by_id.keys())
for family_id in missing_family_ids:
family = create_model_from_json(Family, {'project': project, 'family_id': family_id}, user)
families_by_id[family_id] = family
updated_families.add(family)
individual_models = Individual.objects.filter(family__project=project).prefetch_related(
'family', 'mother', 'father')
has_individual_guid = any(record.get('individualGuid') for record in individual_records)
if has_individual_guid:
individual_lookup = {
i.guid: i for i in individual_models.filter(
guid__in=[record['individualGuid'] for record in individual_records])
}
else:
individual_lookup = defaultdict(dict)
for i in individual_models.filter(
individual_id__in=[_get_record_individual_id(record) for record in individual_records]):
individual_lookup[i.individual_id][i.family] = i
for record in individual_records:
family_id = _get_record_family_id(record)
family = families_by_id.get(family_id)
if has_individual_guid:
individual = individual_lookup[record.pop('individualGuid')]
else:
# uploaded files do not have unique guid's so fall back to a combination of family and individualId
individual_id = _get_record_individual_id(record)
individual = individual_lookup[individual_id].get(family)
if not individual:
individual = create_model_from_json(
Individual, {'family': family, 'individual_id': individual_id, 'case_review_status': 'I'}, user)
record['family'] = family
record.pop('familyId', None)
if individual.family != family:
family = individual.family
updated_families.add(family)
previous_id = record.pop(JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN, None)
if previous_id:
updated_individuals.update(individual.maternal_children.all())
updated_individuals.update(individual.paternal_children.all())
record['displayName'] = ''
# Update the parent ids last, so if they are referencing updated individuals they will check for the correct ID
if record.get('maternalId') or record.get('paternalId'):
parent_updates.append({
'individual': individual,
'maternalId': record.pop('maternalId', None),
'paternalId': record.pop('paternalId', None),
})
family_notes = record.pop(JsonConstants.FAMILY_NOTES_COLUMN, None)
if family_notes:
update_family_from_json(family, {'analysis_notes': family_notes}, user)
updated_families.add(family)
is_updated = update_individual_from_json(individual, record, user=user, allow_unknown_keys=True)
if is_updated:
updated_individuals.add(individual)
updated_families.add(family)
for update in parent_updates:
individual = update.pop('individual')
is_updated = update_individual_from_json(individual, update, user=user)
if is_updated:
updated_individuals.add(individual)
updated_families.add(individual.family)
# update pedigree images
update_pedigree_images(updated_families, user, project_guid=project.guid)
return list(updated_families), list(updated_individuals)
def test_update_pedigree_images(self, mock_tempfile, mock_os_system,
mock_randint):
mock_tempfile.gettempdir.return_value = '/tmp'
mock_tempfile_file = mock_tempfile.NamedTemporaryFile.return_value.__enter__.return_value
mock_tempfile_file.name = 'temp.fam'
mock_randint.return_value = 123456
def _mock_paint(command):
with open(command.split('-outfile ')[-1], 'wb') as f:
f.write(b'\xff\xd8\xff')
mock_os_system.side_effect = _mock_paint
test_families = Family.objects.filter(guid='F000001_1')
update_pedigree_images(test_families)
pedigree_image = test_families.first().pedigree_image
self.assertTrue(bool(pedigree_image))
self.assertEqual(pedigree_image.name,
'pedigree_images/pedigree_image_123456.png')
os.remove(pedigree_image.path)
mock_os_system.assert_called_with(
'perl /seqr/management/commands/HaploPainter1.043.pl -b -outformat png -pedfile temp.fam -family 1 -outfile /tmp/pedigree_image_123456.png'
)
mock_tempfile_file.write.assert_has_calls([
mock.call('\t'.join(
['1', 'NA19675_1', 'NA19678', 'NA19679', '1', '2'])),
mock.call('\n'),
mock.call('\t'.join(['1', 'NA19678', '0', '0', '1', '1'])),
mock.call('\n'),
mock.call('\t'.join(['1', 'NA19679', '0', '0', '2', '1'])),
mock.call('\n'),
])
# Create placeholder when only has one parent
Sample.objects.get(guid='S000130_na19678').delete()
Individual.objects.get(individual_id='NA19678').delete()
mock_tempfile_file.write.reset_mock()
mock_os_system.reset_mock()
update_pedigree_images(test_families)
pedigree_image = test_families.first().pedigree_image
self.assertTrue(bool(pedigree_image))
self.assertEqual(pedigree_image.name,
'pedigree_images/pedigree_image_123456.png')
os.remove(pedigree_image.path)
mock_os_system.assert_called_with(
'perl /seqr/management/commands/HaploPainter1.043.pl -b -outformat png -pedfile temp.fam -family 1 -outfile /tmp/pedigree_image_123456.png'
)
mock_tempfile_file.write.assert_has_calls([
mock.call('\t'.join(
['1', 'NA19675_1', 'placeholder_123456', 'NA19679', '1',
'2'])),
mock.call('\n'),
mock.call('\t'.join(['1', 'NA19679', '0', '0', '2', '1'])),
mock.call('\n'),
mock.call('\t'.join(
['1', 'placeholder_123456', '0', '0', '1', '9'])),
mock.call('\n'),
])
# Do not generate for families with one individual
mock_tempfile_file.write.reset_mock()
mock_os_system.reset_mock()
one_individual_families = Family.objects.filter(guid='F000003_3')
update_pedigree_images(one_individual_families)
pedigree_image = one_individual_families.first().pedigree_image
self.assertFalse(bool(pedigree_image))
mock_os_system.assert_not_called()
mock_tempfile_file.write.assert_not_called()
def _add_or_update_individuals_and_families(project, individual_records, user=None):
"""Add or update individual and family records in the given project.
Args:
project (object): Django ORM model for the project to add families to
individual_records (list): A list of JSON records representing individuals. See
the return value of pedigree_info_utils#convert_fam_file_rows_to_json(..)
Return:
2-tuple: updated_families, updated_individuals containing Django ORM models
"""
families = {}
updated_individuals = set()
parent_updates = []
for i, record in enumerate(individual_records):
# family id will be in different places in the json depending on whether it comes from a flat uploaded file or from the nested individual object
family_id = record.get(JsonConstants.FAMILY_ID_COLUMN) or record.get('family', {}).get('familyId')
if not family_id:
raise ValueError("record #%s doesn't contain a 'familyId' key: %s" % (i, record))
if JsonConstants.INDIVIDUAL_ID_COLUMN not in record and 'individualGuid' not in record:
raise ValueError("record #%s doesn't contain an 'individualId' key: %s" % (i, record))
family = families.get(family_id)
if family:
created = False
else:
family, created = Family.objects.get_or_create(project=project, family_id=family_id)
if created:
logger.info("Created family: %s", family)
# uploaded files do not have unique guid's so fall back to a combination of family and individualId
if record.get('individualGuid'):
individual_filters = {'guid': record['individualGuid']}
else:
individual_id = record.get(JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN) or record[JsonConstants.INDIVIDUAL_ID_COLUMN]
individual_filters = {'family': family, 'individual_id': individual_id}
individual, created = Individual.objects.get_or_create(**individual_filters)
if created:
record.update({
'caseReviewStatus': 'I',
})
record['family'] = family
record.pop('familyId', None)
if individual.family != family:
families[individual.family.family_id] = individual.family
if record.get(JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN):
updated_individuals.update(individual.maternal_children.all())
updated_individuals.update(individual.paternal_children.all())
record['displayName'] = ''
# Update the parent ids last, so if they are referencing updated individuals they will check for the correct ID
if record.get('maternalId') or record.get('paternalId'):
parent_updates.append({
'individual': individual,
'maternalId': record.pop('maternalId', None),
'paternalId': record.pop('paternalId', None),
})
update_individual_from_json(individual, record, allow_unknown_keys=True, user=user)
if record.get(JsonConstants.FAMILY_NOTES_COLUMN):
update_family_from_json(family, {'analysis_notes': record[JsonConstants.FAMILY_NOTES_COLUMN]})
updated_individuals.add(individual)
families[family.family_id] = family
for update in parent_updates:
individual = update.pop('individual')
update_individual_from_json(individual, update, user=user)
updated_families = list(families.values())
# update pedigree images
update_pedigree_images(updated_families, project_guid=project.guid)
return updated_families, list(updated_individuals)