def main(args):
skip_count = 0
new_or_updated_count = 0
matched_count = 0
with open(args.file, 'r') as csv_file:
sites_reader = csv.DictReader(csv_file)
hpo_dao = HPODao()
site_dao = SiteDao()
existing_site_map = {
site.googleGroup: site
for site in site_dao.get_all()
}
with site_dao.session() as session:
for row in sites_reader:
site = _site_from_row(row, hpo_dao)
if site is None:
skip_count += 1
continue
changed = _upsert_site(site,
existing_site_map.get(site.googleGroup),
site_dao, session, args.dry_run)
if changed:
new_or_updated_count += 1
else:
matched_count += 1
logging.info(
'Done%s. %d skipped, %d sites new/updated, %d sites not changed.',
' (dry run)' if args.dry_run else '', skip_count, new_or_updated_count,
matched_count)
def get_site_id_by_site_value(obj):
if 'site' in obj:
from dao.site_dao import SiteDao
site_dao = SiteDao()
site = site_dao.get_by_google_group(obj['site']['value'])
if site is not None:
return site.siteId
return None
def setUp(self, **kwargs):
super(SyncConsentFilesTest, self).setUp(use_mysql=True, **kwargs)
NdbTestBase.doSetUp(self)
TestBase.setup_fake(self)
self.org_dao = OrganizationDao()
self.site_dao = SiteDao()
self.participant_dao = ParticipantDao()
self.summary_dao = ParticipantSummaryDao()
def __init__(self):
super(ParticipantSummaryDao,
self).__init__(ParticipantSummary,
order_by_ending=_ORDER_BY_ENDING)
self.hpo_dao = HPODao()
self.code_dao = CodeDao()
self.site_dao = SiteDao()
self.organization_dao = OrganizationDao()
def _to_json(model, inactive_sites):
resource = _FhirOrganization()
resource.id = model.externalId
resource.display_name = model.displayName
if inactive_sites:
resource.sites = [SiteDao._to_json(site) for site in model.sites]
else:
resource.sites = [
SiteDao._to_json(site) for site in model.sites
if site.siteStatus == site.siteStatus.ACTIVE
]
return resource
def _setup_unset_enrollment_site(self):
site_dao = SiteDao()
organization_dao = OrganizationDao()
org_2 = organization_dao.insert(
Organization(externalId='ORG_2',
displayName='Organization 2',
hpoId=PITT_HPO_ID))
site_dao.insert(
Site(siteName='not enrolling site',
googleGroup='not_enrolling_dot_com',
organizationId=org_2.organizationId,
enrollingStatus=EnrollingStatus.UNSET,
siteStatus=SiteStatus.INACTIVE))
def add_missing_hpo_from_site(self, session, participant_id, site_id):
if site_id is None:
raise BadRequest('No site ID given for auto-pairing participant.')
site = SiteDao().get_with_session(session, site_id)
if site is None:
raise BadRequest('Invalid siteId reference %r.' % site_id)
participant = self.get_for_update(session, participant_id)
if participant is None:
raise BadRequest('No participant %r for HPO ID udpate.' %
participant_id)
if participant.siteId == site.siteId:
return
participant.hpoId = site.hpoId
participant.organizationId = site.organizationId
participant.siteId = site.siteId
participant.providerLink = make_primary_provider_link_for_id(
site.hpoId)
if participant.participantSummary is None:
raise RuntimeError('No ParticipantSummary available for P%d.' %
participant_id)
participant.participantSummary.hpoId = site.hpoId
participant.lastModified = clock.CLOCK.now()
# Update the version and add history row
self._do_update(session, participant, participant)
def test_overwrite_existing_pairing(self):
participant_id = 99
created = self.dao.insert(
Participant(participantId=participant_id,
biobankId=2,
hpoId=self._test_db.hpo_id,
providerLink=make_primary_provider_link_for_id(
self._test_db.hpo_id)))
self.participant_summary_dao.insert(self.participant_summary(created))
self.assertEquals(created.hpoId, self._test_db.hpo_id) # sanity check
other_hpo = HPODao().insert(
HPO(hpoId=PITT_HPO_ID + 1, name='DIFFERENT_HPO'))
other_site = SiteDao().insert(
Site(hpoId=other_hpo.hpoId,
siteName='Arbitrary Site',
googleGroup='*****@*****.**'))
with self.dao.session() as session:
self.dao.add_missing_hpo_from_site(session, participant_id,
other_site.siteId)
# Original Participant + summary is affected.
refetched = self.dao.get(participant_id)
self.assertEquals(refetched.hpoId, other_hpo.hpoId)
self.assertEquals(refetched.providerLink,
make_primary_provider_link_for_id(other_hpo.hpoId))
self.assertEquals(
self.participant_summary_dao.get(participant_id).hpoId,
other_hpo.hpoId)
def setUp(self, **kwargs):
super(MetricsEhrApiTestBase, self).setUp(use_mysql=True, **kwargs)
self.dao = ParticipantDao()
self.ps_dao = ParticipantSummaryDao()
self.ehr_receipt_dao = EhrReceiptDao()
self.ps = ParticipantSummary()
self.calendar_dao = CalendarDao()
self.site_dao = SiteDao()
self.hpo_dao = HPODao()
self.org_dao = OrganizationDao()
self.hpo_test = self._make_hpo(hpoId=TEST_HPO_ID,
name=TEST_HPO_NAME,
displayName='Test',
organizationType=OrganizationType.UNSET)
self.hpo_foo = self._make_hpo(hpoId=10, name='FOO', displayName='Foo')
self.hpo_bar = self._make_hpo(hpoId=11, name='BAR', displayName='Bar')
self.org_foo_a = self._make_org(organizationId=10,
externalId='FOO_A',
displayName='Foo A',
hpoId=self.hpo_foo.hpoId)
self.org_bar_a = self._make_org(organizationId=11,
externalId='BAR_A',
displayName='Bar A',
hpoId=self.hpo_bar.hpoId)
def __init__(self):
args = parser.parse_args()
self.organization_dao = OrganizationDao()
self.stub_geocoding = args.stub_geocoding
self.ACTIVE = SiteStatus.ACTIVE
self.status_exception_list = ['hpo-site-walgreensphoenix']
self.instance = args.instance
self.creds_file = args.creds_file
self.new_sites_list = []
self.project = None
if args.project:
self.project = args.project
if self.project in ENV_LIST:
self.environment = ' ' + self.project.split('-')[-1].upper()
else:
self.environment = ' ' + ENV_TEST.split('-')[-1].upper()
super(SiteImporter,
self).__init__('site', SiteDao(), 'siteId', 'googleGroup', [
SITE_ORGANIZATION_ID_COLUMN, SITE_SITE_ID_COLUMN,
SITE_SITE_COLUMN, SITE_STATUS_COLUMN + self.environment,
ENROLLING_STATUS_COLUMN + self.environment,
DIGITAL_SCHEDULING_STATUS_COLUMN + self.environment
])
def get_location_site_id(location_value):
if not location_value.startswith(_LOCATION_PREFIX):
logging.warn("Invalid location: %s" % location_value)
return None
google_group = location_value[len(_LOCATION_PREFIX):]
site = SiteDao().get_by_google_group(google_group)
if not site:
logging.warn("Unknown site: %s" % google_group)
return None
return site.siteId
def __init__(self):
super(SiteImporter,
self).__init__('site', SiteDao(), 'siteId', 'googleGroup', [
SITE_ORGANIZATION_ID_COLUMN, SITE_SITE_ID_COLUMN,
SITE_SITE_COLUMN, SITE_STATUS_COLUMN, ENROLLING_STATUS_COLUMN
])
self.organization_dao = OrganizationDao()
args = parser.parse_args()
self.geocode_flag = args.geocode_flag
self.ACTIVE = SiteStatus.ACTIVE
self.status_exception_list = ['hpo-site-walgreensphoenix']
def _cleanup_old_entities(self, session, row_list, dry_run):
log_prefix = '(dry run) ' if dry_run else ''
self.site_dao = SiteDao()
existing_sites = set(site.googleGroup
for site in self.site_dao.get_all())
site_group_list_from_sheet = [
str(row[SITE_SITE_ID_COLUMN].lower()) for row in row_list
]
sites_to_remove = existing_sites - set(site_group_list_from_sheet)
if not sites_to_remove:
return
site_id_list = []
for site in sites_to_remove:
old_site = self.site_dao.get_by_google_group(site)
if old_site and old_site.isObsolete != ObsoleteStatus.OBSOLETE:
logging.info(
log_prefix +
'Deleting old Site no longer in Google sheet: %s', site)
site_id_list.append(old_site.siteId)
self.deletion_count += 1
elif old_site and old_site.isObsolete == ObsoleteStatus.OBSOLETE:
logging.info(
'Not attempting to delete site [%s] with existing obsolete status',
old_site.googleGroup)
if dry_run or not site_id_list:
return
# Set obsolete first, in case the sites cannot be deleted below.
str_list = ','.join([str(i) for i in site_id_list])
sql = """ UPDATE site
SET is_obsolete = 1
WHERE site_id in ({site_id_list})""".format(site_id_list=str_list)
session.execute(sql)
self.site_dao._invalidate_cache()
# Try to delete old sites, this may partially fail if there are still foreign references.
self.delete_sql_statement(session, str_list)
def __init__(self, client, use_local_files=None):
self._use_local_files = use_local_files
self._client = client
self._hpos = HPODao().get_all()
self._sites = SiteDao().get_all()
if not self._sites:
raise BadRequest('No sites found; import sites before running generator.')
self._now = clock.CLOCK.now()
self._consent_questionnaire_id_and_version = None
self._setup_data()
self._setup_questionnaires()
self._min_birth_date = self._now - datetime.timedelta(days=_MAX_PARTICIPANT_AGE * 365)
self._max_days_for_birth_date = 365 * (_MAX_PARTICIPANT_AGE - _MIN_PARTICIPANT_AGE)
def _setup_hpos(self, org_dao=None):
hpo_dao = HPODao()
hpo_dao.insert(
HPO(hpoId=UNSET_HPO_ID,
name='UNSET',
displayName='Unset',
organizationType=OrganizationType.UNSET))
hpo_dao.insert(
HPO(hpoId=PITT_HPO_ID,
name='PITT',
displayName='Pittsburgh',
organizationType=OrganizationType.HPO))
hpo_dao.insert(
HPO(hpoId=AZ_HPO_ID,
name='AZ_TUCSON',
displayName='Arizona',
organizationType=OrganizationType.HPO))
self.hpo_id = PITT_HPO_ID
org_dao = OrganizationDao()
org_dao.insert(
Organization(organizationId=AZ_ORG_ID,
externalId='AZ_TUCSON_BANNER_HEALTH',
displayName='Banner Health',
hpoId=AZ_HPO_ID))
created_org = org_dao.insert(
Organization(organizationId=PITT_ORG_ID,
externalId='PITT_BANNER_HEALTH',
displayName='PITT display Banner Health',
hpoId=PITT_HPO_ID))
self.organization_id = created_org.organizationId
site_dao = SiteDao()
created_site = site_dao.insert(
Site(siteName='Monroeville Urgent Care Center',
googleGroup='hpo-site-monroeville',
mayolinkClientNumber=7035769,
organizationId=PITT_ORG_ID,
hpoId=PITT_HPO_ID))
self.site_id = created_site.siteId
site_dao.insert(
Site(siteName='Phoenix Urgent Care Center',
googleGroup='hpo-site-bannerphoenix',
mayolinkClientNumber=7035770,
organizationId=PITT_ORG_ID,
hpoId=PITT_HPO_ID))
site_dao.insert(
Site(siteName='Phoenix clinic',
googleGroup='hpo-site-clinic-phoenix',
mayolinkClientNumber=7035770,
organizationId=AZ_ORG_ID,
hpoId=AZ_HPO_ID))
def _to_handling_info(self, username, site_id):
if not username and not site_id:
return None
info = _FhirBiobankOrderHandlingInfo()
if site_id:
site = SiteDao().get(site_id)
info.site = Identifier()
info.site.system = SITE_ID_SYSTEM
info.site.value = site.googleGroup
if username:
info.author = Identifier()
info.author.system = HEALTHPRO_USERNAME_SYSTEM
info.author.value = username
return info
def _setup_hpos(self):
hpo_dao = HPODao()
hpo_dao.insert(HPO(hpoId=UNSET_HPO_ID, name='UNSET'))
hpo_dao.insert(
HPO(hpoId=PITT_HPO_ID,
name='PITT',
organizationType=OrganizationType.HPO))
self.hpo_id = PITT_HPO_ID
site_dao = SiteDao()
created_site = site_dao.insert(
Site(siteName='Monroeville Urgent Care Center',
googleGroup='hpo-site-monroeville',
consortiumName='Pittsburgh',
mayolinkClientNumber=7035769,
hpoId=PITT_HPO_ID))
self.site_id = created_site.siteId
site_dao.insert(
Site(siteName='Phoenix Urgent Care Center',
googleGroup='hpo-site-bannerphoenix',
consortiumName='Pittsburgh',
mayolinkClientNumber=7035770,
hpoId=PITT_HPO_ID))
def setUp(self):
super(HierarchyContentApiTest, self).setUp(with_data=False)
hpo_dao = HPODao()
hpo_dao.insert(
HPO(hpoId=UNSET_HPO_ID,
name='UNSET',
displayName='Unset',
organizationType=OrganizationType.UNSET,
resourceId='h123456'))
hpo_dao.insert(
HPO(hpoId=PITT_HPO_ID,
name='PITT',
displayName='Pittsburgh',
organizationType=OrganizationType.HPO,
resourceId='h123457'))
hpo_dao.insert(
HPO(hpoId=AZ_HPO_ID,
name='AZ_TUCSON',
displayName='Arizona',
organizationType=OrganizationType.HPO,
resourceId='h123458'))
self.site_dao = SiteDao()
self.org_dao = OrganizationDao()
def _parse_handling_info(self, handling_info):
site_id = None
username = None
if handling_info.site:
if handling_info.site.system != SITE_ID_SYSTEM:
raise BadRequest('Invalid site system: %s' %
handling_info.site.system)
site = SiteDao().get_by_google_group(handling_info.site.value)
if not site:
raise BadRequest('Unrecognized site: %s' %
handling_info.site.value)
site_id = site.siteId
if handling_info.author:
if handling_info.author.system != HEALTHPRO_USERNAME_SYSTEM:
raise BadRequest('Invalid author system: %s' %
handling_info.author.system)
username = handling_info.author.value
return username, site_id
def _setup_data(self):
organization_dao = OrganizationDao()
site_dao = SiteDao()
org_1 = organization_dao.insert(
Organization(externalId='ORG_1',
displayName='Organization 1',
hpoId=PITT_HPO_ID,
resourceId='o123456'))
organization_dao.insert(
Organization(externalId='AARDVARK_ORG',
displayName='Aardvarks Rock',
hpoId=PITT_HPO_ID,
resourceId='o123457'))
site_dao.insert(
Site(siteName='Site 1',
googleGroup='hpo-site-1',
mayolinkClientNumber=123456,
organizationId=org_1.organizationId,
siteStatus=SiteStatus.ACTIVE,
enrollingStatus=EnrollingStatus.ACTIVE,
launchDate=datetime.datetime(2016, 1, 1),
notes='notes',
latitude=12.1,
longitude=13.1,
directions='directions',
physicalLocationName='locationName',
address1='address1',
address2='address2',
city='Austin',
state='TX',
zipCode='78751',
phoneNumber='555-555-5555',
adminEmails='[email protected], [email protected]',
link='http://www.example.com'))
site_dao.insert(
Site(siteName='Zebras Rock',
googleGroup='aaaaaaa',
organizationId=org_1.organizationId,
enrollingStatus=EnrollingStatus.INACTIVE,
siteStatus=SiteStatus.INACTIVE))
def __init__(self):
super(ParticipantDao, self).__init__(Participant)
self.hpo_dao = HPODao()
self.organization_dao = OrganizationDao()
self.site_dao = SiteDao()
class SiteImporter(CsvImporter):
def __init__(self):
args = parser.parse_args()
self.organization_dao = OrganizationDao()
self.stub_geocoding = args.stub_geocoding
self.ACTIVE = SiteStatus.ACTIVE
self.status_exception_list = ['hpo-site-walgreensphoenix']
self.instance = args.instance
self.creds_file = args.creds_file
self.new_sites_list = []
self.project = None
if args.project:
self.project = args.project
if self.project in ENV_LIST:
self.environment = ' ' + self.project.split('-')[-1].upper()
else:
self.environment = ' ' + ENV_TEST.split('-')[-1].upper()
super(SiteImporter,
self).__init__('site', SiteDao(), 'siteId', 'googleGroup', [
SITE_ORGANIZATION_ID_COLUMN, SITE_SITE_ID_COLUMN,
SITE_SITE_COLUMN, SITE_STATUS_COLUMN + self.environment,
ENROLLING_STATUS_COLUMN + self.environment,
DIGITAL_SCHEDULING_STATUS_COLUMN + self.environment
])
def run(self, filename, dry_run):
super(SiteImporter, self).run(filename, dry_run)
insert_participants = False
if not dry_run:
if self.environment:
current_env = ENV_STABLE
if self.environment.strip() == 'STABLE' and len(
self.new_sites_list) > 0:
from googleapiclient.discovery import build
logging.info(
'Starting reboot of app instances to insert new test participants'
)
service = build('appengine', 'v1', cache_discovery=False)
request = service.apps().services().versions().list(
appsId=current_env, servicesId='default')
versions = request.execute()
for version in versions['versions']:
if version['servingStatus'] == 'SERVING':
_id = version['id']
request = service.apps().services().versions(
).instances().list(servicesId='default',
versionsId=_id,
appsId=current_env)
instances = request.execute()
try:
for instance in instances['instances']:
sha = instance['name'].split('/')[-1]
delete_instance = service.apps().services(
).versions().instances().delete(
appsId=current_env,
servicesId='default',
versionsId=_id,
instancesId=sha)
response = delete_instance.execute()
if response['done']:
insert_participants = True
logging.info(
'Reboot of instance: %s in stable complete.',
instance['name'])
else:
logging.warn(
'Not able to reboot instance on server, Error: %s',
response)
except KeyError:
logging.warn('No running instance for %s',
version['name'])
if insert_participants:
logging.info('Starting import of test participants.')
self._insert_new_participants(self.new_sites_list)
def delete_sql_statement(self, session, str_list):
sql = """
DELETE FROM site
WHERE site_id IN ({str_list})
AND NOT EXISTS(
SELECT * FROM participant WHERE site_id = site.site_id)
AND NOT EXISTS(
SELECT * FROM participant_summary WHERE site_id = site.site_id
OR physical_measurements_finalized_site_id = site.site_id
OR physical_measurements_created_site_id = site.site_id
OR biospecimen_source_site_id = site.site_id
OR biospecimen_collected_site_id = site.site_id
OR biospecimen_processed_site_id = site.site_id
OR biospecimen_finalized_site_id = site.site_id
)
AND NOT EXISTS(
SELECT * FROM participant_history WHERE site_id = site.site_id)
AND NOT EXISTS(
SELECT * FROM physical_measurements WHERE created_site_id = site.site_id
OR finalized_site_id = site.site_id)
AND NOT EXISTS(
SELECT * FROM biobank_order WHERE finalized_site_id = site.site_id
OR source_site_id = site.site_id
OR collected_site_id = site.site_id
OR processed_site_id = site.site_id
)
""".format(str_list=str_list)
session.execute(sql)
def _cleanup_old_entities(self, session, row_list, dry_run):
log_prefix = '(dry run) ' if dry_run else ''
self.site_dao = SiteDao()
existing_sites = set(site.googleGroup
for site in self.site_dao.get_all())
site_group_list_from_sheet = [
str(row[SITE_SITE_ID_COLUMN].lower()) for row in row_list
]
sites_to_remove = existing_sites - set(site_group_list_from_sheet)
if sites_to_remove:
site_id_list = []
for site in sites_to_remove:
logging.info(
log_prefix +
'Deleting old Site no longer in Google sheet: %s', site)
old_site = self.site_dao.get_by_google_group(site)
if old_site and old_site.isObsolete != ObsoleteStatus.OBSOLETE:
site_id_list.append(old_site.siteId)
self.deletion_count += 1
elif old_site and old_site.isObsolete == ObsoleteStatus.OBSOLETE:
logging.info(
'Not attempting to delete site [%s] with existing obsolete status',
old_site.googleGroup)
if site_id_list and not dry_run:
str_list = ','.join([str(i) for i in site_id_list])
logging.info(log_prefix + 'Marking old site as obsolete : %s',
old_site)
sql = """ UPDATE site
SET is_obsolete = 1
WHERE site_id in ({site_id_list})""".format(site_id_list=str_list)
session.execute(sql)
self.site_dao._invalidate_cache()
# Try to delete old sites.
self.delete_sql_statement(session, str_list)
def _insert_new_participants(self, entity):
num_participants = 0
participants = {
'zip_code': '20001',
'date_of_birth': '1933-3-3',
'gender_identity': 'GenderIdentity_Woman',
'withdrawalStatus': 'NOT_WITHDRAWN',
'suspensionStatus': 'NOT_SUSPENDED'
}
client = Client('rdr/v1', False, self.creds_file, self.instance)
client_log.setLevel(logging.WARN)
questionnaire_to_questions, consent_questionnaire_id_and_version = _setup_questionnaires(
client)
consent_questions = questionnaire_to_questions[
consent_questionnaire_id_and_version]
for site in entity:
for participant, v in enumerate(range(1, 21), 1):
num_participants += 1
participant = participants
participant.update(
{'last_name': site.googleGroup.split('-')[-1]})
participant.update({'first_name': 'Participant {}'.format(v)})
participant.update({'site': site.googleGroup})
import_participant(participant, client,
consent_questionnaire_id_and_version,
questionnaire_to_questions,
consent_questions, num_participants)
logging.info('%d participants imported.' % num_participants)
def _entity_from_row(self, row):
google_group = row[SITE_SITE_ID_COLUMN].lower()
organization = self.organization_dao.get_by_external_id(
row[SITE_ORGANIZATION_ID_COLUMN].upper())
if organization is None:
logging.warn('Invalid organization ID %s importing site %s',
row[SITE_ORGANIZATION_ID_COLUMN].upper(),
google_group)
self.errors.append(
'Invalid organization ID {} importing site {}'.format(
row[SITE_ORGANIZATION_ID_COLUMN].upper(), google_group))
return None
launch_date = None
launch_date_str = row.get(SITE_LAUNCH_DATE_COLUMN)
if launch_date_str:
try:
launch_date = parse(launch_date_str).date()
except ValueError:
logging.warn('Invalid launch date %s for site %s',
launch_date_str, google_group)
self.errors.append('Invalid launch date {} for site {}'.format(
launch_date_str, google_group))
return None
name = row[SITE_SITE_COLUMN]
mayolink_client_number = None
mayolink_client_number_str = row.get(
SITE_MAYOLINK_CLIENT_NUMBER_COLUMN)
if mayolink_client_number_str:
try:
mayolink_client_number = int(mayolink_client_number_str)
except ValueError:
logging.warn('Invalid Mayolink Client # %s for site %s',
mayolink_client_number_str, google_group)
self.errors.append(
'Invalid Mayolink Client # {} for site {}'.format(
mayolink_client_number_str, google_group))
return None
notes = row.get(SITE_NOTES_COLUMN)
notes_es = row.get(SITE_NOTES_COLUMN_ES)
try:
site_status = SiteStatus(row[SITE_STATUS_COLUMN +
self.environment].upper())
except TypeError:
logging.warn('Invalid site status %s for site %s',
row[SITE_STATUS_COLUMN + self.environment],
google_group)
self.errors.append('Invalid site status {} for site {}'.format(
row[SITE_STATUS_COLUMN + self.environment], google_group))
return None
try:
enrolling_status = EnrollingStatus(row[ENROLLING_STATUS_COLUMN +
self.environment].upper())
except TypeError:
logging.warn('Invalid enrollment site status %s for site %s',
row[ENROLLING_STATUS_COLUMN + self.environment],
google_group)
self.errors.append(
'Invalid enrollment site status {} for site {}'.format(
row[ENROLLING_STATUS_COLUMN + self.environment],
google_group))
directions = row.get(SITE_DIRECTIONS_COLUMN)
physical_location_name = row.get(SITE_PHYSICAL_LOCATION_NAME_COLUMN)
address_1 = row.get(SITE_ADDRESS_1_COLUMN)
address_2 = row.get(SITE_ADDRESS_2_COLUMN)
city = row.get(SITE_CITY_COLUMN)
state = row.get(SITE_STATE_COLUMN)
zip_code = row.get(SITE_ZIP_COLUMN)
phone = row.get(SITE_PHONE_COLUMN)
admin_email_addresses = row.get(SITE_ADMIN_EMAIL_ADDRESSES_COLUMN)
link = row.get(SITE_LINK_COLUMN)
digital_scheduling_status = DigitalSchedulingStatus(
row[DIGITAL_SCHEDULING_STATUS_COLUMN + self.environment].upper())
schedule_instructions = row.get(SCHEDULING_INSTRUCTIONS)
schedule_instructions_es = row.get(SCHEDULING_INSTRUCTIONS_ES)
return Site(siteName=name,
googleGroup=google_group,
mayolinkClientNumber=mayolink_client_number,
organizationId=organization.organizationId,
hpoId=organization.hpoId,
siteStatus=site_status,
enrollingStatus=enrolling_status,
digitalSchedulingStatus=digital_scheduling_status,
scheduleInstructions=schedule_instructions,
scheduleInstructions_ES=schedule_instructions_es,
launchDate=launch_date,
notes=notes,
notes_ES=notes_es,
directions=directions,
physicalLocationName=physical_location_name,
address1=address_1,
address2=address_2,
city=city,
state=state,
zipCode=zip_code,
phoneNumber=phone,
adminEmails=admin_email_addresses,
link=link)
def _update_entity(self, entity, existing_entity, session, dry_run):
self._populate_lat_lng_and_time_zone(entity, existing_entity)
if entity.siteStatus == self.ACTIVE and (entity.latitude == None
or entity.longitude == None):
self.errors.append(
'Skipped active site without geocoding: {}'.format(
entity.googleGroup))
return None, True
return super(SiteImporter,
self)._update_entity(entity, existing_entity, session,
dry_run)
def _insert_entity(self, entity, existing_map, session, dry_run):
self._populate_lat_lng_and_time_zone(entity, None)
if entity.siteStatus == self.ACTIVE and (entity.latitude == None
or entity.longitude == None):
self.errors.append(
'Skipped active site without geocoding: {}'.format(
entity.googleGroup))
return False
self.new_sites_list.append(entity)
super(SiteImporter, self)._insert_entity(entity, existing_map, session,
dry_run)
def _populate_lat_lng_and_time_zone(self, site, existing_site):
if site.address1 and site.city and site.state:
if existing_site:
if (existing_site.address1 == site.address1
and existing_site.city == site.city
and existing_site.state == site.state
and existing_site.latitude is not None
and existing_site.longitude is not None
and existing_site.timeZoneId is not None):
# Address didn't change, use the existing lat/lng and time zone.
site.latitude = existing_site.latitude
site.longitude = existing_site.longitude
site.timeZoneId = existing_site.timeZoneId
return
if self.stub_geocoding:
# Set dummy latitude and longitude when importing sites locally / on a CircleCI box.
site.latitude = 32.176
site.longitude = -110.93
site.timeZoneId = 'America/Phoenix'
else:
latitude, longitude = self._get_lat_long_for_site(
site.address1, site.city, site.state)
site.latitude = latitude
site.longitude = longitude
if latitude and longitude:
site.timeZoneId = self._get_time_zone(latitude, longitude)
else:
if site.googleGroup not in self.status_exception_list:
if site.siteStatus == self.ACTIVE:
self.errors.append(
'Active site must have valid address. Site: {}, Group: {}'
.format(site.siteName, site.googleGroup))
def _get_lat_long_for_site(self, address_1, city, state):
self.full_address = address_1 + ' ' + city + ' ' + state
try:
self.api_key = os.environ.get('API_KEY')
self.gmaps = googlemaps.Client(key=self.api_key)
try:
geocode_result = self.gmaps.geocode(address_1 + '' + city +
' ' + state)[0]
except IndexError:
self.errors.append(
'Bad address for {}, could not geocode.'.format(
self.full_address))
return None, None
if geocode_result:
geometry = geocode_result.get('geometry')
if geometry:
location = geometry.get('location')
if location:
latitude = location.get('lat')
longitude = location.get('lng')
return latitude, longitude
else:
logging.warn('Can not find lat/long for %s',
self.full_address)
self.errors.append('Can not find lat/long for {}'.format(
self.full_address))
return None, None
else:
logging.warn('Geocode results failed for %s.',
self.full_address)
self.errors.append('Geocode results failed for {}'.format(
self.full_address))
return None, None
except ValueError as e:
logging.exception('Invalid geocode key: %s. ERROR: %s',
self.api_key, e)
self.errors.append('Invalid geocode key: {}. ERROR: {}'.format(
self.api_key, e))
return None, None
except IndexError as e:
logging.exception(
'Geocoding failure Check that address is correct. ERROR: %s',
e)
self.errors.append(
'Geocoding failured Check that address is correct. ERROR: {}'.
format(self.api_key, e))
return None, None
def _get_time_zone(self, latitude, longitude):
time_zone = self.gmaps.timezone(location=(latitude, longitude))
if time_zone['status'] == 'OK':
time_zone_id = time_zone['timeZoneId']
return time_zone_id
else:
logging.info('can not retrieve time zone from %s',
self.full_address)
self.errors.append('Can not retrieve time zone from {}'.format(
self.full_address))
return None
def setUp(self):
super(SiteDaoTest, self).setUp()
self.site_dao = SiteDao()
def setUp(self):
super(SiteDaoTest, self).setUp()
self.site_dao = SiteDao()
self.participant_dao = ParticipantDao()
self.ps_dao = ParticipantSummaryDao()
self.ps_history = ParticipantHistoryDao()
class SyncConsentFilesTest(CloudStorageSqlTestBase, NdbTestBase):
"""Tests behavior of sync_consent_files
"""
def setUp(self, **kwargs):
super(SyncConsentFilesTest, self).setUp(use_mysql=True, **kwargs)
NdbTestBase.doSetUp(self)
TestBase.setup_fake(self)
self.org_dao = OrganizationDao()
self.site_dao = SiteDao()
self.participant_dao = ParticipantDao()
self.summary_dao = ParticipantSummaryDao()
def tearDown(self):
super(SyncConsentFilesTest, self).tearDown()
def _create_org(self, id_):
org = Organization(organizationId=id_,
externalId=id_,
displayName=id_,
hpoId=UNSET_HPO_ID)
self.org_dao.insert(org)
return org
def _create_site(self, id_, google_group):
site = Site(siteId=id_, siteName=id_, googleGroup=google_group)
self.site_dao.insert(site)
return site
def _create_participant(self,
id_,
org_id,
site_id,
consents=False,
ghost=None,
email=None,
null_email=False):
participant = Participant(participantId=id_,
biobankId=id_,
organizationId=org_id,
siteId=site_id,
isGhostId=ghost)
self.participant_dao.insert(participant)
summary = self.participant_summary(participant)
if consents:
summary.consentForElectronicHealthRecords = 1
summary.consentForStudyEnrollment = 1
if email:
summary.email = email
if null_email:
summary.email = None
self.summary_dao.insert(summary)
return participant
def test_iter_participants_data(self):
"""should list consenting participants
"""
org1 = self._create_org(1)
org2 = self._create_org(2)
site1 = self._create_site(1001, 'group1')
site2 = self._create_site(1002, 'group2')
self._create_participant(1,
org1.organizationId,
site1.siteId,
consents=True,
null_email=True)
self._create_participant(2, org2.organizationId, site2.siteId)
self._create_participant(3,
org1.organizationId,
None,
consents=True,
ghost=False)
self._create_participant(4,
org1.organizationId,
None,
consents=True,
ghost=True)
self._create_participant(5,
org1.organizationId,
None,
consents=True,
email='*****@*****.**')
participant_data_list = list(
sync_consent_files._iter_participants_data())
participant_ids = [d.participant_id for d in participant_data_list]
self.assertEqual(len(participant_ids), 2,
"finds correct number of results")
self.assertEqual(participant_ids, [1, 3], "finds valid participants")
self.assertEqual(participant_data_list[0].google_group, 'group1',
"Includes google group")
self.assertEqual(participant_data_list[1].google_group, None,
"allows None for google group")
@mock.patch('cloudstorage.listbucket')
@mock.patch('cloudstorage.copy2')
def test_cloudstorage_copy_objects_api_calls(self, mock_copy2,
mock_listbucket):
"""Makes the proper google cloudstorage API calls
"""
mock_listbucket.return_value = [
cloudstorage.common.GCSFileStat('/fake_bucket1/prefix1/foo', 0,
'x', 0),
cloudstorage.common.GCSFileStat('/fake_bucket1/prefix1/bar', 0,
'x', 0),
]
# with trailing slashes
sync_consent_files.cloudstorage_copy_objects('/fake_bucket1/prefix1/',
'/fake_bucket2/prefix2/')
mock_copy2.assert_has_calls([
mock.call('/fake_bucket1/prefix1/foo',
'/fake_bucket2/prefix2/foo'),
mock.call('/fake_bucket1/prefix1/bar',
'/fake_bucket2/prefix2/bar'),
])
# without trailing slashes
sync_consent_files.cloudstorage_copy_objects('/fake_bucket1/prefix1',
'/fake_bucket2/prefix2')
mock_copy2.assert_has_calls([
mock.call('/fake_bucket1/prefix1/foo',
'/fake_bucket2/prefix2/foo'),
mock.call('/fake_bucket1/prefix1/bar',
'/fake_bucket2/prefix2/bar'),
])
@staticmethod
def _write_cloud_object(cloud_filename, contents_str):
with cloudstorage.cloudstorage_api.open(cloud_filename,
mode='w') as cloud_file:
cloud_file.write(contents_str.encode('utf-8'))
def test_cloudstorage_copy_objects_actual(self):
self._write_cloud_object('/fake_bucket1/prefix/x1/foo.txt', 'foo')
self._write_cloud_object('/fake_bucket1/prefix/x1/bar.txt', 'bar')
self._write_cloud_object('/fake_bucket1/prefix/x1/y1/foo.txt', 'foo')
with cloudstorage.cloudstorage_api.open(
'/fake_bucket1/prefix/x1/foo.txt', mode='r') as f:
self.assertEqual(f.read(), 'foo', 'Wrote to cloud storage')
sync_consent_files.cloudstorage_copy_objects(
'/fake_bucket1/prefix/x1/', '/fake_bucket2/prefix/z/x1/')
self.assertEqual([
file_stat.filename
for file_stat in cloudstorage.cloudstorage_api.listbucket(
'/fake_bucket2/prefix/z/x1/')
], [
'/fake_bucket2/prefix/z/x1/bar.txt',
'/fake_bucket2/prefix/z/x1/foo.txt',
'/fake_bucket2/prefix/z/x1/y1/foo.txt',
], "copied all objects")
with cloudstorage.cloudstorage_api.open(
'/fake_bucket2/prefix/z/x1/foo.txt', mode='r') as f:
self.assertEqual(f.read(), 'foo', 'copied contents')
@mock.patch('cloudstorage.copy2')
def test_cloudstorage_copy_objects_only_new_and_changed(self, copy2):
self._write_cloud_object('/fake_bucket1/prefix/x1/foo.txt', 'foo')
self._write_cloud_object('/fake_bucket1/prefix/x1/bar.txt', 'bar')
self._write_cloud_object('/fake_bucket2/prefix/z/x1/foo.txt', 'foo')
self._write_cloud_object('/fake_bucket2/prefix/z/x1/bar.txt', 'baz')
sync_consent_files.cloudstorage_copy_objects(
'/fake_bucket1/prefix/x1/', '/fake_bucket2/prefix/z/x1/')
copy2.assert_called_once_with('/fake_bucket1/prefix/x1/bar.txt',
'/fake_bucket2/prefix/z/x1/bar.txt')
class SiteDaoTest(SqlTestBase):
def setUp(self):
super(SiteDaoTest, self).setUp()
self.site_dao = SiteDao()
self.participant_dao = ParticipantDao()
self.ps_dao = ParticipantSummaryDao()
self.ps_history = ParticipantHistoryDao()
def test_get_no_sites(self):
self.assertIsNone(self.site_dao.get(9999))
self.assertIsNone(
self.site_dao.get_by_google_group('*****@*****.**'))
def test_insert(self):
site = Site(siteName='site',
googleGroup='*****@*****.**',
mayolinkClientNumber=12345,
hpoId=PITT_HPO_ID)
created_site = self.site_dao.insert(site)
new_site = self.site_dao.get(created_site.siteId)
site.siteId = created_site.siteId
self.assertEquals(site.asdict(), new_site.asdict())
self.assertEquals(
site.asdict(),
self.site_dao.get_by_google_group(
'*****@*****.**').asdict())
def test_update(self):
site = Site(siteName='site',
googleGroup='*****@*****.**',
mayolinkClientNumber=12345,
hpoId=PITT_HPO_ID)
created_site = self.site_dao.insert(site)
new_site = Site(siteId=created_site.siteId,
siteName='site2',
googleGroup='*****@*****.**',
mayolinkClientNumber=123456,
hpoId=UNSET_HPO_ID)
self.site_dao.update(new_site)
fetched_site = self.site_dao.get(created_site.siteId)
self.assertEquals(new_site.asdict(), fetched_site.asdict())
self.assertEquals(
new_site.asdict(),
self.site_dao.get_by_google_group(
'*****@*****.**').asdict())
self.assertIsNone(
self.site_dao.get_by_google_group('*****@*****.**'))
def test_participant_pairing_updates_on_change(self):
TIME = datetime.datetime(2018, 1, 1)
TIME2 = datetime.datetime(2018, 1, 2)
provider_link = '[{"organization": {"reference": "Organization/AZ_TUCSON"}, "primary": true}]'
site = Site(siteName='site',
googleGroup='*****@*****.**',
mayolinkClientNumber=12345,
hpoId=PITT_HPO_ID,
organizationId=PITT_ORG_ID)
created_site = self.site_dao.insert(site)
with FakeClock(TIME):
p = Participant(participantId=1,
biobankId=2,
siteId=created_site.siteId)
self.participant_dao.insert(p)
fetch_p = self.participant_dao.get(p.participantId)
updated_p = self.participant_dao.get(fetch_p.participantId)
p_summary = self.ps_dao.insert(self.participant_summary(updated_p))
with FakeClock(TIME2):
update_site_parent = Site(siteId=created_site.siteId,
siteName='site2',
googleGroup='*****@*****.**',
mayolinkClientNumber=123456,
hpoId=AZ_HPO_ID,
organizationId=AZ_ORG_ID)
self.site_dao.update(update_site_parent)
updated_p = self.participant_dao.get(fetch_p.participantId)
ps = self.ps_dao.get(p_summary.participantId)
ph = self.ps_history.get([updated_p.participantId, 1])
self.assertEquals(update_site_parent.hpoId, updated_p.hpoId)
self.assertEquals(update_site_parent.organizationId,
updated_p.organizationId)
self.assertEquals(ps.organizationId, update_site_parent.organizationId)
self.assertEquals(ps.hpoId, update_site_parent.hpoId)
self.assertEquals(ps.organizationId, update_site_parent.organizationId)
self.assertEquals(ph.organizationId, update_site_parent.organizationId)
self.assertEquals(updated_p.providerLink, provider_link)
self.assertEquals(ps.lastModified, TIME2)
class ParticipantSummaryDao(UpdatableDao):
def __init__(self):
super(ParticipantSummaryDao,
self).__init__(ParticipantSummary,
order_by_ending=_ORDER_BY_ENDING)
self.hpo_dao = HPODao()
self.code_dao = CodeDao()
self.site_dao = SiteDao()
self.organization_dao = OrganizationDao()
def get_id(self, obj):
return obj.participantId
def _validate_update(self, session, obj, existing_obj): # pylint: disable=unused-argument
"""Participant summaries don't have a version value; drop it from validation logic."""
if not existing_obj:
raise NotFound('%s with id %s does not exist' %
(self.model_type.__name__, id))
def _has_withdrawn_filter(self, query):
for field_filter in query.field_filters:
if (field_filter.field_name == 'withdrawalStatus'
and field_filter.value == WithdrawalStatus.NO_USE):
return True
if field_filter.field_name == 'withdrawalTime' and field_filter.value is not None:
return True
return False
def _get_non_withdrawn_filter_field(self, query):
"""Returns the first field referenced in query filters which isn't in
WITHDRAWN_PARTICIPANT_FIELDS."""
for field_filter in query.field_filters:
if not field_filter.field_name in WITHDRAWN_PARTICIPANT_FIELDS:
return field_filter.field_name
return None
def _initialize_query(self, session, query_def):
non_withdrawn_field = self._get_non_withdrawn_filter_field(query_def)
if self._has_withdrawn_filter(query_def):
if non_withdrawn_field:
raise BadRequest(
"Can't query on %s for withdrawn participants" %
non_withdrawn_field)
# When querying for withdrawn participants, ensure that the only fields being filtered on or
# ordered by are in WITHDRAWN_PARTICIPANT_FIELDS.
return super(ParticipantSummaryDao,
self)._initialize_query(session, query_def)
else:
query = super(ParticipantSummaryDao,
self)._initialize_query(session, query_def)
if non_withdrawn_field:
# When querying on fields that aren't available for withdrawn participants,
# ensure that we only return participants
# who have not withdrawn or withdrew in the past 48 hours.
withdrawn_visible_start = clock.CLOCK.now(
) - WITHDRAWN_PARTICIPANT_VISIBILITY_TIME
return query.filter(
or_(
ParticipantSummary.withdrawalStatus !=
WithdrawalStatus.NO_USE,
ParticipantSummary.withdrawalTime >=
withdrawn_visible_start))
else:
# When querying on fields that are available for withdrawn participants, return everybody;
# withdrawn participants will have all but WITHDRAWN_PARTICIPANT_FIELDS cleared out 48
# hours after withdrawing.
return query
def _get_order_by_ending(self, query):
if self._has_withdrawn_filter(query):
return _WITHDRAWN_ORDER_BY_ENDING
return self.order_by_ending
def _add_order_by(self, query, order_by, field_names, fields):
if order_by.field_name in _CODE_FILTER_FIELDS:
return super(ParticipantSummaryDao, self)._add_order_by(
query, OrderBy(order_by.field_name + 'Id', order_by.ascending),
field_names, fields)
return super(ParticipantSummaryDao,
self)._add_order_by(query, order_by, field_names, fields)
def make_query_filter(self, field_name, value):
"""Handle HPO and code values when parsing filter values."""
if field_name == 'biobankId':
value = from_client_biobank_id(value, log_exception=True)
if field_name == 'hpoId' or field_name == 'awardee':
hpo = self.hpo_dao.get_by_name(value)
if not hpo:
raise BadRequest('No HPO found with name %s' % value)
if field_name == 'awardee':
field_name = 'hpoId'
return super(ParticipantSummaryDao,
self).make_query_filter(field_name, hpo.hpoId)
if field_name == 'organization':
organization = self.organization_dao.get_by_external_id(value)
if not organization:
raise BadRequest('No organization found with name %s' % value)
return super(ParticipantSummaryDao,
self).make_query_filter(field_name + 'Id',
organization.organizationId)
if field_name in _SITE_FIELDS:
if value == UNSET:
return super(ParticipantSummaryDao,
self).make_query_filter(field_name + 'Id', None)
site = self.site_dao.get_by_google_group(value)
if not site:
raise BadRequest('No site found with google group %s' % value)
return super(ParticipantSummaryDao,
self).make_query_filter(field_name + 'Id',
site.siteId)
if field_name in _CODE_FILTER_FIELDS:
if value == UNSET:
return super(ParticipantSummaryDao,
self).make_query_filter(field_name + 'Id', None)
# Note: we do not at present support querying for UNMAPPED code values.
code = self.code_dao.get_code(PPI_SYSTEM, value)
if not code:
raise BadRequest('No code found: %s' % value)
return super(ParticipantSummaryDao,
self).make_query_filter(field_name + 'Id',
code.codeId)
return super(ParticipantSummaryDao,
self).make_query_filter(field_name, value)
def update_from_biobank_stored_samples(self, participant_id=None):
"""Rewrites sample-related summary data. Call this after updating BiobankStoredSamples.
If participant_id is provided, only that participant will have their summary updated."""
now = clock.CLOCK.now()
sample_sql, sample_params = _get_sample_sql_and_params(now)
baseline_tests_sql, baseline_tests_params = _get_baseline_sql_and_params(
)
dna_tests_sql, dna_tests_params = _get_dna_isolates_sql_and_params()
sample_status_time_sql = _get_sample_status_time_sql_and_params()
sample_status_time_params = {}
counts_sql = """
UPDATE
participant_summary
SET
num_baseline_samples_arrived = {baseline_tests_sql},
samples_to_isolate_dna = {dna_tests_sql},
last_modified = :now
WHERE
num_baseline_samples_arrived != {baseline_tests_sql} OR
samples_to_isolate_dna != {dna_tests_sql}
""".format(baseline_tests_sql=baseline_tests_sql,
dna_tests_sql=dna_tests_sql)
counts_params = {'now': now}
counts_params.update(baseline_tests_params)
counts_params.update(dna_tests_params)
enrollment_status_sql = _ENROLLMENT_STATUS_SQL
enrollment_status_params = {
'submitted': int(QuestionnaireStatus.SUBMITTED),
'unset': int(QuestionnaireStatus.UNSET),
'num_baseline_ppi_modules': self._get_num_baseline_ppi_modules(),
'completed': int(PhysicalMeasurementsStatus.COMPLETED),
'received': int(SampleStatus.RECEIVED),
'full_participant': int(EnrollmentStatus.FULL_PARTICIPANT),
'member': int(EnrollmentStatus.MEMBER),
'interested': int(EnrollmentStatus.INTERESTED),
'now': now
}
# If participant_id is provided, add the participant ID filter to all update statements.
if participant_id:
sample_sql += ' AND participant_id = :participant_id'
sample_params['participant_id'] = participant_id
counts_sql += ' AND participant_id = :participant_id'
counts_params['participant_id'] = participant_id
enrollment_status_sql += ' AND participant_id = :participant_id'
enrollment_status_params['participant_id'] = participant_id
sample_status_time_sql += ' AND a.participant_id = :participant_id'
sample_status_time_params['participant_id'] = participant_id
sample_sql = replace_null_safe_equals(sample_sql)
counts_sql = replace_null_safe_equals(counts_sql)
with self.session() as session:
session.execute(sample_sql, sample_params)
session.execute(counts_sql, counts_params)
session.execute(enrollment_status_sql, enrollment_status_params)
# TODO: Change this to the optimized sql in _update_dv_stored_samples()
session.execute(sample_status_time_sql, sample_status_time_params)
def _get_num_baseline_ppi_modules(self):
return len(
config.getSettingList(config.BASELINE_PPI_QUESTIONNAIRE_FIELDS))
def update_enrollment_status(self, summary):
"""Updates the enrollment status field on the provided participant summary to
the correct value based on the other fields on it. Called after
a questionnaire response or physical measurements are submitted."""
consent = (summary.consentForStudyEnrollment == QuestionnaireStatus.SUBMITTED and
summary.consentForElectronicHealthRecords == QuestionnaireStatus.SUBMITTED) or \
(summary.consentForStudyEnrollment == QuestionnaireStatus.SUBMITTED and
summary.consentForElectronicHealthRecords is None and
summary.consentForDvElectronicHealthRecordsSharing == QuestionnaireStatus.SUBMITTED)
enrollment_status = self.calculate_enrollment_status(
consent, summary.numCompletedBaselinePPIModules,
summary.physicalMeasurementsStatus, summary.samplesToIsolateDNA)
summary.enrollmentStatusMemberTime = self.calculate_member_time(
consent, summary)
summary.enrollmentStatusCoreOrderedSampleTime = self.calculate_core_ordered_sample_time(
consent, summary)
summary.enrollmentStatusCoreStoredSampleTime = self.calculate_core_stored_sample_time(
consent, summary)
# Update last modified date if status changes
if summary.enrollmentStatus != enrollment_status:
summary.lastModified = clock.CLOCK.now()
summary.enrollmentStatus = enrollment_status
def calculate_enrollment_status(self, consent,
num_completed_baseline_ppi_modules,
physical_measurements_status,
samples_to_isolate_dna):
if consent:
if (num_completed_baseline_ppi_modules
== self._get_num_baseline_ppi_modules()
and physical_measurements_status
== PhysicalMeasurementsStatus.COMPLETED
and samples_to_isolate_dna == SampleStatus.RECEIVED):
return EnrollmentStatus.FULL_PARTICIPANT
return EnrollmentStatus.MEMBER
return EnrollmentStatus.INTERESTED
def calculate_member_time(self, consent, participant_summary):
if consent and participant_summary.enrollmentStatusMemberTime is not None:
return participant_summary.enrollmentStatusMemberTime
elif consent:
if participant_summary.consentForElectronicHealthRecords is None and \
participant_summary.consentForDvElectronicHealthRecordsSharing == \
QuestionnaireStatus.SUBMITTED:
return participant_summary.consentForDvElectronicHealthRecordsSharingTime
return participant_summary.consentForElectronicHealthRecordsTime
else:
return None
def calculate_core_stored_sample_time(self, consent, participant_summary):
if consent and \
participant_summary.numCompletedBaselinePPIModules == \
self._get_num_baseline_ppi_modules() and \
participant_summary.physicalMeasurementsStatus == PhysicalMeasurementsStatus.COMPLETED and \
participant_summary.samplesToIsolateDNA == SampleStatus.RECEIVED:
max_core_sample_time = self.calculate_max_core_sample_time(
participant_summary, field_name_prefix='sampleStatus')
if max_core_sample_time and participant_summary.enrollmentStatusCoreStoredSampleTime:
return participant_summary.enrollmentStatusCoreStoredSampleTime
else:
return max_core_sample_time
else:
return None
def calculate_core_ordered_sample_time(self, consent, participant_summary):
if consent and \
participant_summary.numCompletedBaselinePPIModules == \
self._get_num_baseline_ppi_modules() and \
participant_summary.physicalMeasurementsStatus == PhysicalMeasurementsStatus.COMPLETED:
max_core_sample_time = self.calculate_max_core_sample_time(
participant_summary, field_name_prefix='sampleOrderStatus')
if max_core_sample_time and participant_summary.enrollmentStatusCoreOrderedSampleTime:
return participant_summary.enrollmentStatusCoreOrderedSampleTime
else:
return max_core_sample_time
else:
return None
def calculate_max_core_sample_time(self,
participant_summary,
field_name_prefix='sampleStatus'):
keys = [
field_name_prefix + '%sTime' % test
for test in config.getSettingList(config.DNA_SAMPLE_TEST_CODES)
]
sample_time_list = \
[v for k, v in participant_summary if k in keys and v is not None]
sample_time = min(sample_time_list) if sample_time_list else None
if sample_time is not None:
return max([
sample_time, participant_summary.enrollmentStatusMemberTime,
participant_summary.questionnaireOnTheBasicsTime,
participant_summary.questionnaireOnLifestyleTime,
participant_summary.questionnaireOnOverallHealthTime,
participant_summary.physicalMeasurementsFinalizedTime
])
else:
return None
def calculate_distinct_visits(self,
pid,
finalized_time,
id_,
amendment=False):
""" Participants may get PM or biobank samples on same day. This should be considered as
a single visit in terms of program payment to participant.
return Boolean: true if there has not been an order on same date."""
from dao.biobank_order_dao import BiobankOrderDao
from dao.physical_measurements_dao import PhysicalMeasurementsDao
day_has_order, day_has_measurement = False, False
existing_orders = BiobankOrderDao().get_biobank_orders_for_participant(
pid)
ordered_samples = BiobankOrderDao(
).get_ordered_samples_for_participant(pid)
existing_measurements = PhysicalMeasurementsDao(
).get_measuremnets_for_participant(pid)
order_id_to_finalized_date = {
sample.biobankOrderId: sample.finalized.date()
for sample in ordered_samples if sample.finalized
}
if existing_orders and finalized_time:
for order in existing_orders:
order_finalized_date = order_id_to_finalized_date.get(
order.biobankOrderId)
if order_finalized_date == finalized_time.date() and order.biobankOrderId != id_ and \
order.orderStatus != BiobankOrderStatus.CANCELLED:
day_has_order = True
elif order.biobankOrderId == id_ and order.orderStatus == BiobankOrderStatus.AMENDED:
day_has_order = True
elif not finalized_time and amendment:
day_has_order = True
if existing_measurements and finalized_time:
for measurement in existing_measurements:
if not measurement.finalized:
continue
if measurement.finalized.date() == finalized_time.date() and measurement.physicalMeasurementsId\
!= id_:
day_has_measurement = True
is_distinct_visit = not (day_has_order or day_has_measurement)
return is_distinct_visit
def to_client_json(self, model):
result = model.asdict()
# Participants that withdrew more than 48 hours ago should have fields other than
# WITHDRAWN_PARTICIPANT_FIELDS cleared.
if (model.withdrawalStatus == WithdrawalStatus.NO_USE and
(model.withdrawalTime is None or model.withdrawalTime <
clock.CLOCK.now() - WITHDRAWN_PARTICIPANT_VISIBILITY_TIME)):
result = {k: result.get(k) for k in WITHDRAWN_PARTICIPANT_FIELDS}
elif model.withdrawalStatus != WithdrawalStatus.NO_USE and \
model.suspensionStatus == SuspensionStatus.NO_CONTACT:
for i in SUSPENDED_PARTICIPANT_FIELDS:
result[i] = UNSET
result['participantId'] = to_client_participant_id(model.participantId)
biobank_id = result.get('biobankId')
if biobank_id:
result['biobankId'] = to_client_biobank_id(biobank_id)
date_of_birth = result.get('dateOfBirth')
if date_of_birth:
result['ageRange'] = get_bucketed_age(date_of_birth,
clock.CLOCK.now())
else:
result['ageRange'] = UNSET
if result.get('primaryLanguage') is None:
result['primaryLanguage'] = UNSET
if 'organizationId' in result:
result['organization'] = result['organizationId']
del result['organizationId']
format_json_org(result, self.organization_dao, 'organization')
format_json_hpo(result, self.hpo_dao, 'hpoId')
result['awardee'] = result['hpoId']
_initialize_field_type_sets()
for fieldname in _DATE_FIELDS:
format_json_date(result, fieldname)
for fieldname in _CODE_FIELDS:
format_json_code(result, self.code_dao, fieldname)
for fieldname in _ENUM_FIELDS:
format_json_enum(result, fieldname)
for fieldname in _SITE_FIELDS:
format_json_site(result, self.site_dao, fieldname)
if (model.withdrawalStatus == WithdrawalStatus.NO_USE
or model.suspensionStatus == SuspensionStatus.NO_CONTACT):
result['recontactMethod'] = 'NO_CONTACT'
# Strip None values.
result = {k: v for k, v in result.iteritems() if v is not None}
return result
def _decode_token(self, query_def, fields):
""" If token exists in participant_summary api, decode and use lastModified to add a buffer
of 60 seconds. This ensures when a _sync link is used no one is missed. This will return
at a minimum, the last participant and any more that have been modified in the previous 60
seconds. Duplicate participants returned should be handled on the client side."""
decoded_vals = super(ParticipantSummaryDao,
self)._decode_token(query_def, fields)
if query_def.order_by and (query_def.order_by.field_name
== 'lastModified'
and query_def.always_return_token is True
and query_def.backfill_sync is True):
decoded_vals[0] = decoded_vals[0] - datetime.timedelta(
seconds=config.LAST_MODIFIED_BUFFER_SECONDS)
return decoded_vals
@staticmethod
def update_ehr_status(summary, update_time):
summary.ehrStatus = EhrStatus.PRESENT
if not summary.ehrReceiptTime:
summary.ehrReceiptTime = update_time
summary.ehrUpdateTime = update_time
return summary
class SiteDaoTest(SqlTestBase):
def setUp(self):
super(SiteDaoTest, self).setUp()
self.site_dao = SiteDao()
def test_get_no_sites(self):
self.assertIsNone(self.site_dao.get(9999))
self.assertIsNone(
self.site_dao.get_by_google_group('*****@*****.**'))
def test_insert(self):
site = Site(siteName='site',
googleGroup='*****@*****.**',
consortiumName='consortium',
mayolinkClientNumber=12345,
hpoId=PITT_HPO_ID)
created_site = self.site_dao.insert(site)
new_site = self.site_dao.get(created_site.siteId)
site.siteId = created_site.siteId
self.assertEquals(site.asdict(), new_site.asdict())
self.assertEquals(
site.asdict(),
self.site_dao.get_by_google_group(
'*****@*****.**').asdict())
def test_update(self):
site = Site(siteName='site',
googleGroup='*****@*****.**',
consortiumName='consortium',
mayolinkClientNumber=12345,
hpoId=PITT_HPO_ID)
created_site = self.site_dao.insert(site)
new_site = Site(siteId=created_site.siteId,
siteName='site2',
googleGroup='*****@*****.**',
consortiumName='consortium2',
mayolinkClientNumber=123456,
hpoId=UNSET_HPO_ID)
self.site_dao.update(new_site)
fetched_site = self.site_dao.get(created_site.siteId)
self.assertEquals(new_site.asdict(), fetched_site.asdict())
self.assertEquals(
new_site.asdict(),
self.site_dao.get_by_google_group(
'*****@*****.**').asdict())
self.assertIsNone(
self.site_dao.get_by_google_group('*****@*****.**'))
class ParticipantDao(UpdatableDao):
def __init__(self):
super(ParticipantDao, self).__init__(Participant)
self.hpo_dao = HPODao()
self.organization_dao = OrganizationDao()
self.site_dao = SiteDao()
def get_id(self, obj):
return obj.participantId
def insert_with_session(self, session, obj):
obj.hpoId = self._get_hpo_id(obj)
obj.version = 1
obj.signUpTime = clock.CLOCK.now().replace(microsecond=0)
obj.lastModified = obj.signUpTime
if obj.withdrawalStatus is None:
obj.withdrawalStatus = WithdrawalStatus.NOT_WITHDRAWN
if obj.suspensionStatus is None:
obj.suspensionStatus = SuspensionStatus.NOT_SUSPENDED
super(ParticipantDao, self).insert_with_session(session, obj)
history = ParticipantHistory()
history.fromdict(obj.asdict(), allow_pk=True)
session.add(history)
return obj
def insert(self, obj):
if obj.participantId:
assert obj.biobankId
return super(ParticipantDao, self).insert(obj)
assert not obj.biobankId
return self._insert_with_random_id(obj, ('participantId', 'biobankId'))
def update_ghost_participant(self, session, pid):
if not pid:
raise Forbidden('Can not update participant without id')
participant = self.get_for_update(session, pid)
if participant is None:
logging.warn('Tried to mark participant with id: [%r] as ghost but participant does not'
'exist. Wrong environment?' % pid)
else:
participant.isGhostId = 1
participant.dateAddedGhost = clock.CLOCK.now()
self._update_history(session, participant, participant)
super(ParticipantDao, self)._do_update(session, participant, participant)
def _check_if_external_id_exists(self, obj):
with self.session() as session:
return session.query(Participant).filter_by(externalId=obj.externalId).first()
def _update_history(self, session, obj, existing_obj):
# Increment the version and add a new history entry.
obj.version = existing_obj.version + 1
history = ParticipantHistory()
history.fromdict(obj.asdict(), allow_pk=True)
session.add(history)
def _validate_update(self, session, obj, existing_obj):
# Withdrawal and suspension have default values assigned on insert, so they should always have
# explicit values in updates.
if obj.withdrawalStatus is None:
raise BadRequest('missing withdrawal status in update')
if obj.suspensionStatus is None:
raise BadRequest('missing suspension status in update')
if obj.withdrawalReason != WithdrawalReason.UNSET and obj.withdrawalReason is not None and \
obj.withdrawalReasonJustification is None:
raise BadRequest('missing withdrawalReasonJustification in update')
super(ParticipantDao, self)._validate_update(session, obj, existing_obj)
# Once a participant marks their withdrawal status as NO_USE, it can't be changed back.
# TODO: Consider the future ability to un-withdraw.
if (existing_obj.withdrawalStatus == WithdrawalStatus.NO_USE
and obj.withdrawalStatus != WithdrawalStatus.NO_USE):
raise Forbidden('Participant %d has withdrawn, cannot unwithdraw' % obj.participantId)
def get_for_update(self, session, obj_id):
# Fetch the participant summary at the same time as the participant, as we are potentially
# updating both.
return self.get_with_session(session, obj_id, for_update=True,
options=joinedload(Participant.participantSummary))
def _do_update(self, session, obj, existing_obj):
"""Updates the associated ParticipantSummary, and extracts HPO ID from the provider link
or set pairing at another level (site/organization/awardee) with parent/child enforcement."""
obj.lastModified = clock.CLOCK.now()
obj.signUpTime = existing_obj.signUpTime
obj.biobankId = existing_obj.biobankId
obj.withdrawalTime = existing_obj.withdrawalTime
obj.suspensionTime = existing_obj.suspensionTime
need_new_summary = False
if obj.withdrawalStatus != existing_obj.withdrawalStatus:
obj.withdrawalTime = (obj.lastModified if obj.withdrawalStatus == WithdrawalStatus.NO_USE
else None)
obj.withdrawalAuthored = obj.withdrawalAuthored \
if obj.withdrawalStatus == WithdrawalStatus.NO_USE else None
need_new_summary = True
if obj.suspensionStatus != existing_obj.suspensionStatus:
obj.suspensionTime = (obj.lastModified if obj.suspensionStatus == SuspensionStatus.NO_CONTACT
else None)
need_new_summary = True
update_pairing = True
if obj.siteId is None and obj.organizationId is None and obj.hpoId is None and \
obj.providerLink == 'null':
# Prevent unpairing if /PUT is sent with no pairing levels.
update_pairing = False
if update_pairing is True:
has_id = False
if obj.organizationId or obj.siteId or (obj.hpoId >= 0):
has_id = True
provider_link_unchanged = True
if obj.providerLink is not None:
if existing_obj.providerLink:
provider_link_unchanged = json.loads(obj.providerLink) == \
json.loads(existing_obj.providerLink)
else:
provider_link_unchanged = False
null_provider_link = obj.providerLink == 'null'
# site,org,or awardee is sent in request: Get relationships and try to set provider link.
if has_id and (provider_link_unchanged or null_provider_link):
site, organization, awardee = self.get_pairing_level(obj)
obj.organizationId = organization
obj.siteId = site
obj.hpoId = awardee
if awardee is not None and (obj.hpoId != existing_obj.hpoId):
# get provider link for hpo_id (awardee)
obj.providerLink = make_primary_provider_link_for_id(awardee)
need_new_summary = True
else: # providerLink has changed
# If the provider link changes, update the HPO ID on the participant and its summary.
if obj.hpoId is None:
obj.hpoId = existing_obj.hpoId
new_hpo_id = self._get_hpo_id(obj)
if new_hpo_id != existing_obj.hpoId:
obj.hpoId = new_hpo_id
obj.siteId = None
obj.organizationId = None
need_new_summary = True
# No pairing updates sent, keep existing values.
if update_pairing == False:
obj.siteId = existing_obj.siteId
obj.organizationId = existing_obj.organizationId
obj.hpoId = existing_obj.hpoId
obj.providerLink = existing_obj.providerLink
if need_new_summary and existing_obj.participantSummary:
# Copy the existing participant summary, and mutate the fields that
# come from participant.
summary = existing_obj.participantSummary
summary.hpoId = obj.hpoId
summary.organizationId = obj.organizationId
summary.siteId = obj.siteId
summary.withdrawalStatus = obj.withdrawalStatus
summary.withdrawalReason = obj.withdrawalReason
summary.withdrawalReasonJustification = obj.withdrawalReasonJustification
summary.withdrawalTime = obj.withdrawalTime
summary.withdrawalAuthored = obj.withdrawalAuthored
summary.suspensionStatus = obj.suspensionStatus
summary.suspensionTime = obj.suspensionTime
summary.lastModified = clock.CLOCK.now()
make_transient(summary)
make_transient(obj)
obj.participantSummary = summary
self._update_history(session, obj, existing_obj)
super(ParticipantDao, self)._do_update(session, obj, existing_obj)
def get_pairing_level(self, obj):
organization_id = obj.organizationId
site_id = obj.siteId
awardee_id = obj.hpoId
# TODO: DO WE WANT TO PREVENT PAIRING IF EXISTING SITE HAS PM/BIO.
if site_id != UNSET and site_id is not None:
site = self.site_dao.get(site_id)
if site is None:
raise BadRequest('Site with site id %s does not exist.' % site_id)
organization_id = site.organizationId
awardee_id = site.hpoId
return site_id, organization_id, awardee_id
elif organization_id != UNSET and organization_id is not None:
organization = self.organization_dao.get(organization_id)
if organization is None:
raise BadRequest('Organization with id %s does not exist.' % organization_id)
awardee_id = organization.hpoId
return None, organization_id, awardee_id
return None, None, awardee_id
@staticmethod
def create_summary_for_participant(obj):
return ParticipantSummary(
participantId=obj.participantId,
lastModified=obj.lastModified,
biobankId=obj.biobankId,
signUpTime=obj.signUpTime,
hpoId=obj.hpoId,
organizationId=obj.organizationId,
siteId=obj.siteId,
withdrawalStatus=obj.withdrawalStatus,
withdrawalReason=obj.withdrawalReason,
withdrawalReasonJustification=obj.withdrawalReasonJustification,
suspensionStatus=obj.suspensionStatus,
enrollmentStatus=EnrollmentStatus.INTERESTED,
ehrStatus=EhrStatus.NOT_PRESENT)
@staticmethod
def _get_hpo_id(obj):
hpo_name = _get_hpo_name_from_participant(obj)
if hpo_name:
hpo = HPODao().get_by_name(hpo_name)
if not hpo:
raise BadRequest('No HPO found with name %s' % hpo_name)
return hpo.hpoId
else:
return UNSET_HPO_ID
def validate_participant_reference(self, session, obj):
"""Raises BadRequest if an object has a missing or invalid participantId reference,
or if the participant has a withdrawal status of NO_USE."""
if obj.participantId is None:
raise BadRequest('%s.participantId required.' % obj.__class__.__name__)
return self.validate_participant_id(session, obj.participantId)
def validate_participant_id(self, session, participant_id):
"""Raises BadRequest if a participant ID is invalid,
or if the participant has a withdrawal status of NO_USE."""
participant = self.get_with_session(session, participant_id)
if participant is None:
raise BadRequest('Participant with ID %d is not found.' % participant_id)
raise_if_withdrawn(participant)
return participant
def get_biobank_ids_sample(self, session, percentage, batch_size):
"""Returns biobank ID and signUpTime for a percentage of participants.
Used in generating fake biobank samples."""
return (session.query(Participant.biobankId, Participant.signUpTime)
.filter(Participant.biobankId % 100 <= percentage * 100)
.yield_per(batch_size))
def to_client_json(self, model):
client_json = {
'participantId': to_client_participant_id(model.participantId),
'externalId': model.externalId,
'hpoId': model.hpoId,
'awardee': model.hpoId,
'organization': model.organizationId,
'siteId': model.siteId,
'biobankId': to_client_biobank_id(model.biobankId),
'lastModified': model.lastModified.isoformat(),
'signUpTime': model.signUpTime.isoformat(),
'providerLink': json.loads(model.providerLink),
'withdrawalStatus': model.withdrawalStatus,
'withdrawalReason': model.withdrawalReason,
'withdrawalReasonJustification': model.withdrawalReasonJustification,
'withdrawalTime': model.withdrawalTime,
'withdrawalAuthored': model.withdrawalAuthored,
'suspensionStatus': model.suspensionStatus,
'suspensionTime': model.suspensionTime
}
format_json_hpo(client_json, self.hpo_dao, 'hpoId'),
format_json_org(client_json, self.organization_dao, 'organization'),
format_json_site(client_json, self.site_dao, 'site'),
format_json_enum(client_json, 'withdrawalStatus')
format_json_enum(client_json, 'withdrawalReason')
format_json_enum(client_json, 'suspensionStatus')
format_json_date(client_json, 'withdrawalTime')
format_json_date(client_json, 'suspensionTime')
client_json['awardee'] = client_json['hpoId']
if 'siteId' in client_json:
del client_json['siteId']
return client_json
def from_client_json(self, resource_json, id_=None, expected_version=None, client_id=None):
parse_json_enum(resource_json, 'withdrawalStatus', WithdrawalStatus)
parse_json_enum(resource_json, 'withdrawalReason', WithdrawalReason)
parse_json_enum(resource_json, 'suspensionStatus', SuspensionStatus)
if 'withdrawalTimeStamp' in resource_json and resource_json['withdrawalTimeStamp'] is not None:
try:
resource_json['withdrawalTimeStamp'] = datetime.datetime\
.utcfromtimestamp(float(resource_json['withdrawalTimeStamp'])/1000)
except (ValueError, TypeError):
raise ValueError("Could not parse {} as TIMESTAMP"
.format(resource_json['withdrawalTimeStamp']))
# biobankId, lastModified, signUpTime are set by DAO.
return Participant(
participantId=id_,
externalId=resource_json.get('externalId'),
version=expected_version,
providerLink=json.dumps(resource_json.get('providerLink')),
clientId=client_id,
withdrawalStatus=resource_json.get('withdrawalStatus'),
withdrawalReason=resource_json.get('withdrawalReason'),
withdrawalAuthored=resource_json.get('withdrawalTimeStamp'),
withdrawalReasonJustification=resource_json.get('withdrawalReasonJustification'),
suspensionStatus=resource_json.get('suspensionStatus'),
organizationId=get_organization_id_from_external_id(resource_json, self.organization_dao),
hpoId=get_awardee_id_from_name(resource_json, self.hpo_dao),
siteId=get_site_id_from_google_group(resource_json, self.site_dao))
def add_missing_hpo_from_site(self, session, participant_id, site_id):
if site_id is None:
raise BadRequest('No site ID given for auto-pairing participant.')
site = SiteDao().get_with_session(session, site_id)
if site is None:
raise BadRequest('Invalid siteId reference %r.' % site_id)
participant = self.get_for_update(session, participant_id)
if participant is None:
raise BadRequest('No participant %r for HPO ID udpate.' % participant_id)
if participant.siteId == site.siteId:
return
participant.hpoId = site.hpoId
participant.organizationId = site.organizationId
participant.siteId = site.siteId
participant.providerLink = make_primary_provider_link_for_id(site.hpoId)
if participant.participantSummary is None:
raise RuntimeError('No ParticipantSummary available for P%d.' % participant_id)
participant.participantSummary.hpoId = site.hpoId
participant.lastModified = clock.CLOCK.now()
# Update the version and add history row
self._do_update(session, participant, participant)
def switch_to_test_account(self, session, participant):
test_hpo_id = HPODao().get_by_name(TEST_HPO_NAME).hpoId
if participant is None:
raise BadRequest('No participant %r for HPO ID udpate.')
if participant.hpoId == test_hpo_id:
return
participant.hpoId = test_hpo_id
participant.organizationId = None
participant.siteId = None
# Update the version and add history row
self._do_update(session, participant, participant)
def handle_integrity_error(self, tried_ids, e, obj):
if 'external_id' in e.message:
existing_participant = self._check_if_external_id_exists(obj)
if existing_participant:
return existing_participant
return super(ParticipantDao, self).handle_integrity_error(tried_ids, e, obj)
class HierarchyContentApiTest(FlaskTestBase):
def setUp(self):
super(HierarchyContentApiTest, self).setUp(with_data=False)
hpo_dao = HPODao()
hpo_dao.insert(
HPO(hpoId=UNSET_HPO_ID,
name='UNSET',
displayName='Unset',
organizationType=OrganizationType.UNSET,
resourceId='h123456'))
hpo_dao.insert(
HPO(hpoId=PITT_HPO_ID,
name='PITT',
displayName='Pittsburgh',
organizationType=OrganizationType.HPO,
resourceId='h123457'))
hpo_dao.insert(
HPO(hpoId=AZ_HPO_ID,
name='AZ_TUCSON',
displayName='Arizona',
organizationType=OrganizationType.HPO,
resourceId='h123458'))
self.site_dao = SiteDao()
self.org_dao = OrganizationDao()
def test_create_new_hpo(self):
self._setup_data()
request_json = {
"resourceType":
"Organization",
"id":
"a893282c-2717-4a20-b276-d5c9c2c0e51f",
'meta': {
'versionId': '1'
},
"extension": [{
"url": "http://all-of-us.org/fhir/sites/awardee-type",
"valueString": "DV"
}],
"identifier": [{
"system": "http://all-of-us.org/fhir/sites/awardee-id",
"value": "TEST_HPO_NAME"
}],
"active":
True,
"type": [{
"coding": [{
"code": "AWARDEE",
"system": "http://all-of-us.org/fhir/sites/type"
}]
}],
"name":
"Test new HPO display name"
}
self.send_put('organization/hierarchy', request_data=request_json)
result = self.send_get('Awardee/TEST_HPO_NAME')
self.assertEquals(
_make_awardee_resource('TEST_HPO_NAME',
'Test new HPO display name', 'DV'), result)
def test_update_existing_hpo(self):
self._setup_data()
request_json = {
"resourceType":
"Organization",
"id":
"a893282c-2717-4a20-b276-d5c9c2c0e51f",
'meta': {
'versionId': '2'
},
"extension": [{
"url": "http://all-of-us.org/fhir/sites/awardee-type",
"valueString": "DV"
}],
"identifier": [{
"system": "http://all-of-us.org/fhir/sites/awardee-id",
"value": "PITT"
}],
"active":
True,
"type": [{
"coding": [{
"code": "AWARDEE",
"system": "http://all-of-us.org/fhir/sites/type"
}]
}],
"name":
"Test update HPO display name"
}
self.send_put('organization/hierarchy', request_data=request_json)
result = self.send_get('Awardee/PITT')
self.assertEqual(result['displayName'], 'Test update HPO display name')
self.assertEqual(result['type'], 'DV')
def test_create_new_organization(self):
self._setup_data()
request_json = {
"resourceType":
"Organization",
"id":
"a893282c-2717-4a20-b276-d5c9c2c0e51f",
'meta': {
'versionId': '1'
},
"extension": [],
"identifier": [{
"system": "http://all-of-us.org/fhir/sites/organization-id",
"value": "TEST_NEW_ORG"
}],
"active":
True,
"type": [{
"coding": [{
"code": "ORGANIZATION",
"system": "http://all-of-us.org/fhir/sites/type"
}]
}],
"name":
"Test create organization display name",
"partOf": {
"reference": "Organization/h123457"
}
}
result_before = self.send_get('Awardee/PITT')
self.assertEqual(2, len(result_before['organizations']))
self.send_put('organization/hierarchy', request_data=request_json)
result_after = self.send_get('Awardee/PITT')
self.assertEqual(3, len(result_after['organizations']))
self.assertIn(
{
'displayName': 'Test create organization display name',
'id': 'TEST_NEW_ORG'
}, result_after['organizations'])
def test_update_existing_organization(self):
self._setup_data()
request_json = {
"resourceType":
"Organization",
"id":
"a893282c-2717-4a20-b276-d5c9c2c0e51f",
'meta': {
'versionId': '2'
},
"extension": [],
"identifier": [{
"system": "http://all-of-us.org/fhir/sites/organization-id",
"value": "AARDVARK_ORG"
}],
"active":
True,
"type": [{
"coding": [{
"code": "ORGANIZATION",
"system": "http://all-of-us.org/fhir/sites/type"
}]
}],
"name":
"Test update organization display name",
"partOf": {
"reference": "Organization/h123457"
}
}
result_before = self.send_get('Awardee/PITT')
self.assertEqual(2, len(result_before['organizations']))
self.send_put('organization/hierarchy', request_data=request_json)
result_after = self.send_get('Awardee/PITT')
self.assertEqual(2, len(result_after['organizations']))
self.assertIn(
{
'displayName': 'Test update organization display name',
'id': 'AARDVARK_ORG'
}, result_after['organizations'])
@mock.patch(
'dao.organization_hierarchy_sync_dao.OrganizationHierarchySyncDao.'
'_get_lat_long_for_site')
@mock.patch(
'dao.organization_hierarchy_sync_dao.OrganizationHierarchySyncDao.'
'_get_time_zone')
def test_create_new_site(self, time_zone, lat_long):
self._setup_data()
lat_long.return_value = 100, 110
time_zone.return_value = 'America/Los_Angeles'
request_json = {
"resourceType":
"Organization",
"id":
"a893282c-2717-4a20-b276-d5c9c2c0e51f",
'meta': {
'versionId': '1'
},
"extension": [{
"url":
"http://all-of-us.org/fhir/sites/enrollmentStatusActive",
"valueBoolean": True
}, {
"url":
"http://all-of-us.org/fhir/sites/digitalSchedulingStatusActive",
"valueBoolean": True
}, {
"url":
"http://all-of-us.org/fhir/sites/schedulingStatusActive",
"valueBoolean": True
}, {
"url": "http://all-of-us.org/fhir/sites/notes",
"valueString": "This is a note about an organization"
}, {
"url":
"http://all-of-us.org/fhir/sites/schedulingInstructions",
"valueString":
"Please schedule appointments up to a week before intended date."
}, {
"url": "http://all-of-us.org/fhir/sites/anticipatedLaunchDate",
"valueDate": "07-02-2010"
}, {
"url": "http://all-of-us.org/fhir/sites/locationName",
"valueString": "Thompson Building"
}, {
"url":
"http://all-of-us.org/fhir/sites/directions",
"valueString":
"Exit 95 N and make a left onto Fake Street"
}],
"identifier": [{
"system": "http://all-of-us.org/fhir/sites/site-id",
"value": "hpo-site-awesome-testing"
}, {
"system":
"http://all-of-us.org/fhir/sites/mayo-link-identifier",
"value": "123456"
}, {
"system":
"http://all-of-us.org/fhir/sites/google-group-identifier",
"value": "Awesome Genomics Testing"
}],
"active":
True,
"type": [{
"coding": [{
"code": "SITE",
"system": "http://all-of-us.org/fhir/sites/type"
}]
}],
"name":
"Awesome Genomics Testing",
"partOf": {
"reference": "Organization/o123457"
},
"address": [{
"line": ["1855 4th Street", "AAC5/6"],
"city": "San Francisco",
"state": "CA",
"postalCode": "94158"
}],
"contact": [{
"telecom": [{
"system": "phone",
"value": "7031234567"
}]
}, {
"telecom": [{
"system": "email",
"value": "*****@*****.**"
}]
}, {
"telecom": [{
"system": "url",
"value": "http://awesome-genomic-testing.com"
}]
}]
}
self.send_put('organization/hierarchy', request_data=request_json)
self.send_get('Awardee/PITT')
existing_map = {
entity.googleGroup: entity
for entity in self.site_dao.get_all()
}
existing_entity = existing_map.get('hpo-site-awesome-testing')
self.assertEqual(existing_entity.adminEmails,
'*****@*****.**')
self.assertEqual(existing_entity.siteStatus, SiteStatus('ACTIVE'))
self.assertEqual(existing_entity.isObsolete, None)
self.assertEqual(existing_entity.city, 'San Francisco')
self.assertEqual(existing_entity.googleGroup,
'hpo-site-awesome-testing')
self.assertEqual(existing_entity.state, 'CA')
self.assertEqual(existing_entity.digitalSchedulingStatus,
DigitalSchedulingStatus('ACTIVE'))
self.assertEqual(existing_entity.mayolinkClientNumber, 123456)
self.assertEqual(existing_entity.address1, '1855 4th Street')
self.assertEqual(existing_entity.address2, 'AAC5/6')
self.assertEqual(existing_entity.zipCode, '94158')
self.assertEqual(existing_entity.directions,
'Exit 95 N and make a left onto Fake Street')
self.assertEqual(existing_entity.notes,
'This is a note about an organization')
self.assertEqual(existing_entity.enrollingStatus,
EnrollingStatus('ACTIVE'))
self.assertEqual(
existing_entity.scheduleInstructions,
'Please schedule appointments up to a week before intended date.')
self.assertEqual(existing_entity.physicalLocationName,
'Thompson Building')
self.assertEqual(existing_entity.link,
'http://awesome-genomic-testing.com')
self.assertEqual(existing_entity.launchDate, datetime.date(2010, 7, 2))
self.assertEqual(existing_entity.phoneNumber, '7031234567')
@mock.patch(
'dao.organization_hierarchy_sync_dao.OrganizationHierarchySyncDao.'
'_get_lat_long_for_site')
@mock.patch(
'dao.organization_hierarchy_sync_dao.OrganizationHierarchySyncDao.'
'_get_time_zone')
def test_update_existing_site(self, time_zone, lat_long):
self._setup_data()
lat_long.return_value = 100, 110
time_zone.return_value = 'America/Los_Angeles'
request_json = {
"resourceType":
"Organization",
"id":
"a893282c-2717-4a20-b276-d5c9c2c0e51f",
'meta': {
'versionId': '2'
},
"extension": [{
"url":
"http://all-of-us.org/fhir/sites/enrollmentStatusActive",
"valueBoolean": True
}, {
"url":
"http://all-of-us.org/fhir/sites/digitalSchedulingStatusActive",
"valueBoolean": False
}, {
"url":
"http://all-of-us.org/fhir/sites/schedulingStatusActive",
"valueBoolean": True
}, {
"url": "http://all-of-us.org/fhir/sites/notes",
"valueString": "This is a note about an organization"
}, {
"url":
"http://all-of-us.org/fhir/sites/schedulingInstructions",
"valueString":
"Please schedule appointments up to a week before intended date."
}, {
"url": "http://all-of-us.org/fhir/sites/anticipatedLaunchDate",
"valueDate": "07-02-2010"
}, {
"url": "http://all-of-us.org/fhir/sites/locationName",
"valueString": "Thompson Building"
}, {
"url":
"http://all-of-us.org/fhir/sites/directions",
"valueString":
"Exit 95 N and make a left onto Fake Street"
}],
"identifier": [{
"system": "http://all-of-us.org/fhir/sites/site-id",
"value": "hpo-site-1"
}, {
"system":
"http://all-of-us.org/fhir/sites/mayo-link-identifier",
"value": "123456"
}, {
"system":
"http://all-of-us.org/fhir/sites/google-group-identifier",
"value": "Awesome Genomics Testing"
}],
"active":
True,
"type": [{
"coding": [{
"code": "SITE",
"system": "http://all-of-us.org/fhir/sites/type"
}]
}],
"name":
"Awesome Genomics Testing",
"partOf": {
"reference": "Organization/o123456"
},
"address": [{
"line": ["1855 4th Street", "AAC5/6"],
"city": "San Francisco",
"state": "CA",
"postalCode": "94158"
}],
"contact": [{
"telecom": [{
"system": "phone",
"value": "7031234567"
}]
}, {
"telecom": [{
"system": "email",
"value": "*****@*****.**"
}]
}, {
"telecom": [{
"system": "url",
"value": "http://awesome-genomic-testing.com"
}]
}]
}
self.send_put('organization/hierarchy', request_data=request_json)
existing_map = {
entity.googleGroup: entity
for entity in self.site_dao.get_all()
}
existing_entity = existing_map.get('hpo-site-1')
self.assertEqual(existing_entity.adminEmails,
'*****@*****.**')
self.assertEqual(existing_entity.siteStatus, SiteStatus('ACTIVE'))
self.assertEqual(existing_entity.isObsolete, None)
self.assertEqual(existing_entity.city, 'San Francisco')
self.assertEqual(existing_entity.googleGroup, 'hpo-site-1')
self.assertEqual(existing_entity.state, 'CA')
self.assertEqual(existing_entity.digitalSchedulingStatus,
DigitalSchedulingStatus('INACTIVE'))
self.assertEqual(existing_entity.mayolinkClientNumber, 123456)
self.assertEqual(existing_entity.address1, '1855 4th Street')
self.assertEqual(existing_entity.address2, 'AAC5/6')
self.assertEqual(existing_entity.zipCode, '94158')
self.assertEqual(existing_entity.directions,
'Exit 95 N and make a left onto Fake Street')
self.assertEqual(existing_entity.notes,
'This is a note about an organization')
self.assertEqual(existing_entity.enrollingStatus,
EnrollingStatus('ACTIVE'))
self.assertEqual(
existing_entity.scheduleInstructions,
'Please schedule appointments up to a week before intended date.')
self.assertEqual(existing_entity.physicalLocationName,
'Thompson Building')
self.assertEqual(existing_entity.link,
'http://awesome-genomic-testing.com')
self.assertEqual(existing_entity.launchDate, datetime.date(2010, 7, 2))
self.assertEqual(existing_entity.phoneNumber, '7031234567')
@mock.patch(
'dao.organization_hierarchy_sync_dao.OrganizationHierarchySyncDao.'
'_get_lat_long_for_site')
@mock.patch(
'dao.organization_hierarchy_sync_dao.OrganizationHierarchySyncDao.'
'_get_time_zone')
def test_create_hpo_org_site(self, time_zone, lat_long):
hpo_json = {
"resourceType":
"Organization",
"id":
"a893282c-2717-4a20-b276-d5c9c2c0e51f",
'meta': {
'versionId': '1'
},
"extension": [{
"url": "http://all-of-us.org/fhir/sites/awardee-type",
"valueString": "DV"
}],
"identifier": [{
"system": "http://all-of-us.org/fhir/sites/awardee-id",
"value": "TEST_HPO_NAME"
}],
"active":
True,
"type": [{
"coding": [{
"code": "AWARDEE",
"system": "http://all-of-us.org/fhir/sites/type"
}]
}],
"name":
"Test new HPO display name"
}
self.send_put('organization/hierarchy', request_data=hpo_json)
org_json = {
"resourceType":
"Organization",
"id":
"a893282c-2717-4a20-b276-d5c9c2c0e123",
'meta': {
'versionId': '1'
},
"extension": [],
"identifier": [{
"system": "http://all-of-us.org/fhir/sites/organization-id",
"value": "TEST_NEW_ORG"
}],
"active":
True,
"type": [{
"coding": [{
"code": "ORGANIZATION",
"system": "http://all-of-us.org/fhir/sites/type"
}]
}],
"name":
"Test create organization display name",
"partOf": {
"reference":
"Organization/a893282c-2717-4a20-b276-d5c9c2c0e51f"
}
}
self.send_put('organization/hierarchy', request_data=org_json)
lat_long.return_value = 100, 110
time_zone.return_value = 'America/Los_Angeles'
site_json = {
"resourceType":
"Organization",
"id":
"a893282c-2717-4a20-b276-d5c9c2c0e234",
'meta': {
'versionId': '1'
},
"extension": [{
"url":
"http://all-of-us.org/fhir/sites/enrollmentStatusActive",
"valueBoolean": True
}, {
"url":
"http://all-of-us.org/fhir/sites/digitalSchedulingStatusActive",
"valueBoolean": True
}, {
"url":
"http://all-of-us.org/fhir/sites/schedulingStatusActive",
"valueBoolean": True
}, {
"url": "http://all-of-us.org/fhir/sites/notes",
"valueString": "This is a note about an organization"
}, {
"url":
"http://all-of-us.org/fhir/sites/schedulingInstructions",
"valueString":
"Please schedule appointments up to a week before intended date."
}, {
"url": "http://all-of-us.org/fhir/sites/anticipatedLaunchDate",
"valueDate": "07-02-2010"
}, {
"url": "http://all-of-us.org/fhir/sites/locationName",
"valueString": "Thompson Building"
}, {
"url":
"http://all-of-us.org/fhir/sites/directions",
"valueString":
"Exit 95 N and make a left onto Fake Street"
}],
"identifier": [{
"system": "http://all-of-us.org/fhir/sites/site-id",
"value": "hpo-site-awesome-testing"
}, {
"system":
"http://all-of-us.org/fhir/sites/mayo-link-identifier",
"value": "123456"
}, {
"system":
"http://all-of-us.org/fhir/sites/google-group-identifier",
"value": "Awesome Genomics Testing"
}],
"active":
True,
"type": [{
"coding": [{
"code": "SITE",
"system": "http://all-of-us.org/fhir/sites/type"
}]
}],
"name":
"Awesome Genomics Testing",
"partOf": {
"reference":
"Organization/a893282c-2717-4a20-b276-d5c9c2c0e123"
},
"address": [{
"line": ["1855 4th Street", "AAC5/6"],
"city": "San Francisco",
"state": "CA",
"postalCode": "94158"
}],
"contact": [{
"telecom": [{
"system": "phone",
"value": "7031234567"
}]
}, {
"telecom": [{
"system": "email",
"value": "*****@*****.**"
}]
}, {
"telecom": [{
"system": "url",
"value": "http://awesome-genomic-testing.com"
}]
}]
}
self.send_put('organization/hierarchy', request_data=site_json)
result = self.send_get('Awardee/TEST_HPO_NAME')
self.assertEqual(
{
u'displayName':
u'Test new HPO display name',
u'type':
u'DV',
u'id':
u'TEST_HPO_NAME',
u'organizations': [{
u'displayName':
u'Test create organization display name',
u'id':
u'TEST_NEW_ORG',
u'sites': [{
u'mayolinkClientNumber':
123456,
u'timeZoneId':
u'America/Los_Angeles',
u'displayName':
u'Awesome Genomics Testing',
u'notes':
u'This is a note about an organization',
u'launchDate':
u'2010-07-02',
u'notesEs':
u'',
u'enrollingStatus':
u'ACTIVE',
u'longitude':
110.0,
u'schedulingInstructions':
u'Please schedule appointments up '
u'to a week before intended date.',
u'latitude':
100.0,
u'physicalLocationName':
u'Thompson Building',
u'phoneNumber':
u'7031234567',
u'siteStatus':
u'ACTIVE',
u'address': {
u'postalCode': u'94158',
u'city': u'San Francisco',
u'line': [u'1855 4th Street', u'AAC5/6'],
u'state': u'CA'
},
u'directions':
u'Exit 95 N and make a left onto Fake Street',
u'link':
u'http://awesome-genomic-testing.com',
u'id':
u'hpo-site-awesome-testing',
u'adminEmails': [u'*****@*****.**'],
u'digitalSchedulingStatus':
u'ACTIVE'
}]
}]
}, result)
def _setup_data(self):
organization_dao = OrganizationDao()
site_dao = SiteDao()
org_1 = organization_dao.insert(
Organization(externalId='ORG_1',
displayName='Organization 1',
hpoId=PITT_HPO_ID,
resourceId='o123456'))
organization_dao.insert(
Organization(externalId='AARDVARK_ORG',
displayName='Aardvarks Rock',
hpoId=PITT_HPO_ID,
resourceId='o123457'))
site_dao.insert(
Site(siteName='Site 1',
googleGroup='hpo-site-1',
mayolinkClientNumber=123456,
organizationId=org_1.organizationId,
siteStatus=SiteStatus.ACTIVE,
enrollingStatus=EnrollingStatus.ACTIVE,
launchDate=datetime.datetime(2016, 1, 1),
notes='notes',
latitude=12.1,
longitude=13.1,
directions='directions',
physicalLocationName='locationName',
address1='address1',
address2='address2',
city='Austin',
state='TX',
zipCode='78751',
phoneNumber='555-555-5555',
adminEmails='[email protected], [email protected]',
link='http://www.example.com'))
site_dao.insert(
Site(siteName='Zebras Rock',
googleGroup='aaaaaaa',
organizationId=org_1.organizationId,
enrollingStatus=EnrollingStatus.INACTIVE,
siteStatus=SiteStatus.INACTIVE))