class SiteDaoTest(SqlTestBase):
def setUp(self):
super(SiteDaoTest, self).setUp()
self.site_dao = SiteDao()
def test_get_no_sites(self):
self.assertIsNone(self.site_dao.get(9999))
self.assertIsNone(
self.site_dao.get_by_google_group('*****@*****.**'))
def test_insert(self):
site = Site(siteName='site',
googleGroup='*****@*****.**',
consortiumName='consortium',
mayolinkClientNumber=12345,
hpoId=PITT_HPO_ID)
created_site = self.site_dao.insert(site)
new_site = self.site_dao.get(created_site.siteId)
site.siteId = created_site.siteId
self.assertEquals(site.asdict(), new_site.asdict())
self.assertEquals(
site.asdict(),
self.site_dao.get_by_google_group(
'*****@*****.**').asdict())
def test_update(self):
site = Site(siteName='site',
googleGroup='*****@*****.**',
consortiumName='consortium',
mayolinkClientNumber=12345,
hpoId=PITT_HPO_ID)
created_site = self.site_dao.insert(site)
new_site = Site(siteId=created_site.siteId,
siteName='site2',
googleGroup='*****@*****.**',
consortiumName='consortium2',
mayolinkClientNumber=123456,
hpoId=UNSET_HPO_ID)
self.site_dao.update(new_site)
fetched_site = self.site_dao.get(created_site.siteId)
self.assertEquals(new_site.asdict(), fetched_site.asdict())
self.assertEquals(
new_site.asdict(),
self.site_dao.get_by_google_group(
'*****@*****.**').asdict())
self.assertIsNone(
self.site_dao.get_by_google_group('*****@*****.**'))
class ParticipantDao(UpdatableDao):
def __init__(self):
super(ParticipantDao, self).__init__(Participant)
self.hpo_dao = HPODao()
self.organization_dao = OrganizationDao()
self.site_dao = SiteDao()
def get_id(self, obj):
return obj.participantId
def insert_with_session(self, session, obj):
obj.hpoId = self._get_hpo_id(obj)
obj.version = 1
obj.signUpTime = clock.CLOCK.now().replace(microsecond=0)
obj.lastModified = obj.signUpTime
if obj.withdrawalStatus is None:
obj.withdrawalStatus = WithdrawalStatus.NOT_WITHDRAWN
if obj.suspensionStatus is None:
obj.suspensionStatus = SuspensionStatus.NOT_SUSPENDED
super(ParticipantDao, self).insert_with_session(session, obj)
history = ParticipantHistory()
history.fromdict(obj.asdict(), allow_pk=True)
session.add(history)
return obj
def insert(self, obj):
if obj.participantId:
assert obj.biobankId
return super(ParticipantDao, self).insert(obj)
assert not obj.biobankId
return self._insert_with_random_id(obj, ('participantId', 'biobankId'))
def update_ghost_participant(self, session, pid):
if not pid:
raise Forbidden('Can not update participant without id')
participant = self.get_for_update(session, pid)
if participant is None:
logging.warn('Tried to mark participant with id: [%r] as ghost but participant does not'
'exist. Wrong environment?' % pid)
else:
participant.isGhostId = 1
participant.dateAddedGhost = clock.CLOCK.now()
self._update_history(session, participant, participant)
super(ParticipantDao, self)._do_update(session, participant, participant)
def _check_if_external_id_exists(self, obj):
with self.session() as session:
return session.query(Participant).filter_by(externalId=obj.externalId).first()
def _update_history(self, session, obj, existing_obj):
# Increment the version and add a new history entry.
obj.version = existing_obj.version + 1
history = ParticipantHistory()
history.fromdict(obj.asdict(), allow_pk=True)
session.add(history)
def _validate_update(self, session, obj, existing_obj):
# Withdrawal and suspension have default values assigned on insert, so they should always have
# explicit values in updates.
if obj.withdrawalStatus is None:
raise BadRequest('missing withdrawal status in update')
if obj.suspensionStatus is None:
raise BadRequest('missing suspension status in update')
if obj.withdrawalReason != WithdrawalReason.UNSET and obj.withdrawalReason is not None and \
obj.withdrawalReasonJustification is None:
raise BadRequest('missing withdrawalReasonJustification in update')
super(ParticipantDao, self)._validate_update(session, obj, existing_obj)
# Once a participant marks their withdrawal status as NO_USE, it can't be changed back.
# TODO: Consider the future ability to un-withdraw.
if (existing_obj.withdrawalStatus == WithdrawalStatus.NO_USE
and obj.withdrawalStatus != WithdrawalStatus.NO_USE):
raise Forbidden('Participant %d has withdrawn, cannot unwithdraw' % obj.participantId)
def get_for_update(self, session, obj_id):
# Fetch the participant summary at the same time as the participant, as we are potentially
# updating both.
return self.get_with_session(session, obj_id, for_update=True,
options=joinedload(Participant.participantSummary))
def _do_update(self, session, obj, existing_obj):
"""Updates the associated ParticipantSummary, and extracts HPO ID from the provider link
or set pairing at another level (site/organization/awardee) with parent/child enforcement."""
obj.lastModified = clock.CLOCK.now()
obj.signUpTime = existing_obj.signUpTime
obj.biobankId = existing_obj.biobankId
obj.withdrawalTime = existing_obj.withdrawalTime
obj.suspensionTime = existing_obj.suspensionTime
need_new_summary = False
if obj.withdrawalStatus != existing_obj.withdrawalStatus:
obj.withdrawalTime = (obj.lastModified if obj.withdrawalStatus == WithdrawalStatus.NO_USE
else None)
obj.withdrawalAuthored = obj.withdrawalAuthored \
if obj.withdrawalStatus == WithdrawalStatus.NO_USE else None
need_new_summary = True
if obj.suspensionStatus != existing_obj.suspensionStatus:
obj.suspensionTime = (obj.lastModified if obj.suspensionStatus == SuspensionStatus.NO_CONTACT
else None)
need_new_summary = True
update_pairing = True
if obj.siteId is None and obj.organizationId is None and obj.hpoId is None and \
obj.providerLink == 'null':
# Prevent unpairing if /PUT is sent with no pairing levels.
update_pairing = False
if update_pairing is True:
has_id = False
if obj.organizationId or obj.siteId or (obj.hpoId >= 0):
has_id = True
provider_link_unchanged = True
if obj.providerLink is not None:
if existing_obj.providerLink:
provider_link_unchanged = json.loads(obj.providerLink) == \
json.loads(existing_obj.providerLink)
else:
provider_link_unchanged = False
null_provider_link = obj.providerLink == 'null'
# site,org,or awardee is sent in request: Get relationships and try to set provider link.
if has_id and (provider_link_unchanged or null_provider_link):
site, organization, awardee = self.get_pairing_level(obj)
obj.organizationId = organization
obj.siteId = site
obj.hpoId = awardee
if awardee is not None and (obj.hpoId != existing_obj.hpoId):
# get provider link for hpo_id (awardee)
obj.providerLink = make_primary_provider_link_for_id(awardee)
need_new_summary = True
else: # providerLink has changed
# If the provider link changes, update the HPO ID on the participant and its summary.
if obj.hpoId is None:
obj.hpoId = existing_obj.hpoId
new_hpo_id = self._get_hpo_id(obj)
if new_hpo_id != existing_obj.hpoId:
obj.hpoId = new_hpo_id
obj.siteId = None
obj.organizationId = None
need_new_summary = True
# No pairing updates sent, keep existing values.
if update_pairing == False:
obj.siteId = existing_obj.siteId
obj.organizationId = existing_obj.organizationId
obj.hpoId = existing_obj.hpoId
obj.providerLink = existing_obj.providerLink
if need_new_summary and existing_obj.participantSummary:
# Copy the existing participant summary, and mutate the fields that
# come from participant.
summary = existing_obj.participantSummary
summary.hpoId = obj.hpoId
summary.organizationId = obj.organizationId
summary.siteId = obj.siteId
summary.withdrawalStatus = obj.withdrawalStatus
summary.withdrawalReason = obj.withdrawalReason
summary.withdrawalReasonJustification = obj.withdrawalReasonJustification
summary.withdrawalTime = obj.withdrawalTime
summary.withdrawalAuthored = obj.withdrawalAuthored
summary.suspensionStatus = obj.suspensionStatus
summary.suspensionTime = obj.suspensionTime
summary.lastModified = clock.CLOCK.now()
make_transient(summary)
make_transient(obj)
obj.participantSummary = summary
self._update_history(session, obj, existing_obj)
super(ParticipantDao, self)._do_update(session, obj, existing_obj)
def get_pairing_level(self, obj):
organization_id = obj.organizationId
site_id = obj.siteId
awardee_id = obj.hpoId
# TODO: DO WE WANT TO PREVENT PAIRING IF EXISTING SITE HAS PM/BIO.
if site_id != UNSET and site_id is not None:
site = self.site_dao.get(site_id)
if site is None:
raise BadRequest('Site with site id %s does not exist.' % site_id)
organization_id = site.organizationId
awardee_id = site.hpoId
return site_id, organization_id, awardee_id
elif organization_id != UNSET and organization_id is not None:
organization = self.organization_dao.get(organization_id)
if organization is None:
raise BadRequest('Organization with id %s does not exist.' % organization_id)
awardee_id = organization.hpoId
return None, organization_id, awardee_id
return None, None, awardee_id
@staticmethod
def create_summary_for_participant(obj):
return ParticipantSummary(
participantId=obj.participantId,
lastModified=obj.lastModified,
biobankId=obj.biobankId,
signUpTime=obj.signUpTime,
hpoId=obj.hpoId,
organizationId=obj.organizationId,
siteId=obj.siteId,
withdrawalStatus=obj.withdrawalStatus,
withdrawalReason=obj.withdrawalReason,
withdrawalReasonJustification=obj.withdrawalReasonJustification,
suspensionStatus=obj.suspensionStatus,
enrollmentStatus=EnrollmentStatus.INTERESTED,
ehrStatus=EhrStatus.NOT_PRESENT)
@staticmethod
def _get_hpo_id(obj):
hpo_name = _get_hpo_name_from_participant(obj)
if hpo_name:
hpo = HPODao().get_by_name(hpo_name)
if not hpo:
raise BadRequest('No HPO found with name %s' % hpo_name)
return hpo.hpoId
else:
return UNSET_HPO_ID
def validate_participant_reference(self, session, obj):
"""Raises BadRequest if an object has a missing or invalid participantId reference,
or if the participant has a withdrawal status of NO_USE."""
if obj.participantId is None:
raise BadRequest('%s.participantId required.' % obj.__class__.__name__)
return self.validate_participant_id(session, obj.participantId)
def validate_participant_id(self, session, participant_id):
"""Raises BadRequest if a participant ID is invalid,
or if the participant has a withdrawal status of NO_USE."""
participant = self.get_with_session(session, participant_id)
if participant is None:
raise BadRequest('Participant with ID %d is not found.' % participant_id)
raise_if_withdrawn(participant)
return participant
def get_biobank_ids_sample(self, session, percentage, batch_size):
"""Returns biobank ID and signUpTime for a percentage of participants.
Used in generating fake biobank samples."""
return (session.query(Participant.biobankId, Participant.signUpTime)
.filter(Participant.biobankId % 100 <= percentage * 100)
.yield_per(batch_size))
def to_client_json(self, model):
client_json = {
'participantId': to_client_participant_id(model.participantId),
'externalId': model.externalId,
'hpoId': model.hpoId,
'awardee': model.hpoId,
'organization': model.organizationId,
'siteId': model.siteId,
'biobankId': to_client_biobank_id(model.biobankId),
'lastModified': model.lastModified.isoformat(),
'signUpTime': model.signUpTime.isoformat(),
'providerLink': json.loads(model.providerLink),
'withdrawalStatus': model.withdrawalStatus,
'withdrawalReason': model.withdrawalReason,
'withdrawalReasonJustification': model.withdrawalReasonJustification,
'withdrawalTime': model.withdrawalTime,
'withdrawalAuthored': model.withdrawalAuthored,
'suspensionStatus': model.suspensionStatus,
'suspensionTime': model.suspensionTime
}
format_json_hpo(client_json, self.hpo_dao, 'hpoId'),
format_json_org(client_json, self.organization_dao, 'organization'),
format_json_site(client_json, self.site_dao, 'site'),
format_json_enum(client_json, 'withdrawalStatus')
format_json_enum(client_json, 'withdrawalReason')
format_json_enum(client_json, 'suspensionStatus')
format_json_date(client_json, 'withdrawalTime')
format_json_date(client_json, 'suspensionTime')
client_json['awardee'] = client_json['hpoId']
if 'siteId' in client_json:
del client_json['siteId']
return client_json
def from_client_json(self, resource_json, id_=None, expected_version=None, client_id=None):
parse_json_enum(resource_json, 'withdrawalStatus', WithdrawalStatus)
parse_json_enum(resource_json, 'withdrawalReason', WithdrawalReason)
parse_json_enum(resource_json, 'suspensionStatus', SuspensionStatus)
if 'withdrawalTimeStamp' in resource_json and resource_json['withdrawalTimeStamp'] is not None:
try:
resource_json['withdrawalTimeStamp'] = datetime.datetime\
.utcfromtimestamp(float(resource_json['withdrawalTimeStamp'])/1000)
except (ValueError, TypeError):
raise ValueError("Could not parse {} as TIMESTAMP"
.format(resource_json['withdrawalTimeStamp']))
# biobankId, lastModified, signUpTime are set by DAO.
return Participant(
participantId=id_,
externalId=resource_json.get('externalId'),
version=expected_version,
providerLink=json.dumps(resource_json.get('providerLink')),
clientId=client_id,
withdrawalStatus=resource_json.get('withdrawalStatus'),
withdrawalReason=resource_json.get('withdrawalReason'),
withdrawalAuthored=resource_json.get('withdrawalTimeStamp'),
withdrawalReasonJustification=resource_json.get('withdrawalReasonJustification'),
suspensionStatus=resource_json.get('suspensionStatus'),
organizationId=get_organization_id_from_external_id(resource_json, self.organization_dao),
hpoId=get_awardee_id_from_name(resource_json, self.hpo_dao),
siteId=get_site_id_from_google_group(resource_json, self.site_dao))
def add_missing_hpo_from_site(self, session, participant_id, site_id):
if site_id is None:
raise BadRequest('No site ID given for auto-pairing participant.')
site = SiteDao().get_with_session(session, site_id)
if site is None:
raise BadRequest('Invalid siteId reference %r.' % site_id)
participant = self.get_for_update(session, participant_id)
if participant is None:
raise BadRequest('No participant %r for HPO ID udpate.' % participant_id)
if participant.siteId == site.siteId:
return
participant.hpoId = site.hpoId
participant.organizationId = site.organizationId
participant.siteId = site.siteId
participant.providerLink = make_primary_provider_link_for_id(site.hpoId)
if participant.participantSummary is None:
raise RuntimeError('No ParticipantSummary available for P%d.' % participant_id)
participant.participantSummary.hpoId = site.hpoId
participant.lastModified = clock.CLOCK.now()
# Update the version and add history row
self._do_update(session, participant, participant)
def switch_to_test_account(self, session, participant):
test_hpo_id = HPODao().get_by_name(TEST_HPO_NAME).hpoId
if participant is None:
raise BadRequest('No participant %r for HPO ID udpate.')
if participant.hpoId == test_hpo_id:
return
participant.hpoId = test_hpo_id
participant.organizationId = None
participant.siteId = None
# Update the version and add history row
self._do_update(session, participant, participant)
def handle_integrity_error(self, tried_ids, e, obj):
if 'external_id' in e.message:
existing_participant = self._check_if_external_id_exists(obj)
if existing_participant:
return existing_participant
return super(ParticipantDao, self).handle_integrity_error(tried_ids, e, obj)
class SiteDaoTest(SqlTestBase):
def setUp(self):
super(SiteDaoTest, self).setUp()
self.site_dao = SiteDao()
self.participant_dao = ParticipantDao()
self.ps_dao = ParticipantSummaryDao()
self.ps_history = ParticipantHistoryDao()
def test_get_no_sites(self):
self.assertIsNone(self.site_dao.get(9999))
self.assertIsNone(
self.site_dao.get_by_google_group('*****@*****.**'))
def test_insert(self):
site = Site(siteName='site',
googleGroup='*****@*****.**',
mayolinkClientNumber=12345,
hpoId=PITT_HPO_ID)
created_site = self.site_dao.insert(site)
new_site = self.site_dao.get(created_site.siteId)
site.siteId = created_site.siteId
self.assertEquals(site.asdict(), new_site.asdict())
self.assertEquals(
site.asdict(),
self.site_dao.get_by_google_group(
'*****@*****.**').asdict())
def test_update(self):
site = Site(siteName='site',
googleGroup='*****@*****.**',
mayolinkClientNumber=12345,
hpoId=PITT_HPO_ID)
created_site = self.site_dao.insert(site)
new_site = Site(siteId=created_site.siteId,
siteName='site2',
googleGroup='*****@*****.**',
mayolinkClientNumber=123456,
hpoId=UNSET_HPO_ID)
self.site_dao.update(new_site)
fetched_site = self.site_dao.get(created_site.siteId)
self.assertEquals(new_site.asdict(), fetched_site.asdict())
self.assertEquals(
new_site.asdict(),
self.site_dao.get_by_google_group(
'*****@*****.**').asdict())
self.assertIsNone(
self.site_dao.get_by_google_group('*****@*****.**'))
def test_participant_pairing_updates_on_change(self):
TIME = datetime.datetime(2018, 1, 1)
TIME2 = datetime.datetime(2018, 1, 2)
provider_link = '[{"organization": {"reference": "Organization/AZ_TUCSON"}, "primary": true}]'
site = Site(siteName='site',
googleGroup='*****@*****.**',
mayolinkClientNumber=12345,
hpoId=PITT_HPO_ID,
organizationId=PITT_ORG_ID)
created_site = self.site_dao.insert(site)
with FakeClock(TIME):
p = Participant(participantId=1,
biobankId=2,
siteId=created_site.siteId)
self.participant_dao.insert(p)
fetch_p = self.participant_dao.get(p.participantId)
updated_p = self.participant_dao.get(fetch_p.participantId)
p_summary = self.ps_dao.insert(self.participant_summary(updated_p))
with FakeClock(TIME2):
update_site_parent = Site(siteId=created_site.siteId,
siteName='site2',
googleGroup='*****@*****.**',
mayolinkClientNumber=123456,
hpoId=AZ_HPO_ID,
organizationId=AZ_ORG_ID)
self.site_dao.update(update_site_parent)
updated_p = self.participant_dao.get(fetch_p.participantId)
ps = self.ps_dao.get(p_summary.participantId)
ph = self.ps_history.get([updated_p.participantId, 1])
self.assertEquals(update_site_parent.hpoId, updated_p.hpoId)
self.assertEquals(update_site_parent.organizationId,
updated_p.organizationId)
self.assertEquals(ps.organizationId, update_site_parent.organizationId)
self.assertEquals(ps.hpoId, update_site_parent.hpoId)
self.assertEquals(ps.organizationId, update_site_parent.organizationId)
self.assertEquals(ph.organizationId, update_site_parent.organizationId)
self.assertEquals(updated_p.providerLink, provider_link)
self.assertEquals(ps.lastModified, TIME2)