def main(args): skip_count = 0 new_or_updated_count = 0 matched_count = 0 with open(args.file, 'r') as csv_file: sites_reader = csv.DictReader(csv_file) hpo_dao = HPODao() site_dao = SiteDao() existing_site_map = { site.googleGroup: site for site in site_dao.get_all() } with site_dao.session() as session: for row in sites_reader: site = _site_from_row(row, hpo_dao) if site is None: skip_count += 1 continue changed = _upsert_site(site, existing_site_map.get(site.googleGroup), site_dao, session, args.dry_run) if changed: new_or_updated_count += 1 else: matched_count += 1 logging.info( 'Done%s. %d skipped, %d sites new/updated, %d sites not changed.', ' (dry run)' if args.dry_run else '', skip_count, new_or_updated_count, matched_count)
def get_site_id_by_site_value(obj): if 'site' in obj: from dao.site_dao import SiteDao site_dao = SiteDao() site = site_dao.get_by_google_group(obj['site']['value']) if site is not None: return site.siteId return None
def setUp(self, **kwargs): super(SyncConsentFilesTest, self).setUp(use_mysql=True, **kwargs) NdbTestBase.doSetUp(self) TestBase.setup_fake(self) self.org_dao = OrganizationDao() self.site_dao = SiteDao() self.participant_dao = ParticipantDao() self.summary_dao = ParticipantSummaryDao()
def __init__(self): super(ParticipantSummaryDao, self).__init__(ParticipantSummary, order_by_ending=_ORDER_BY_ENDING) self.hpo_dao = HPODao() self.code_dao = CodeDao() self.site_dao = SiteDao() self.organization_dao = OrganizationDao()
def _to_json(model, inactive_sites): resource = _FhirOrganization() resource.id = model.externalId resource.display_name = model.displayName if inactive_sites: resource.sites = [SiteDao._to_json(site) for site in model.sites] else: resource.sites = [ SiteDao._to_json(site) for site in model.sites if site.siteStatus == site.siteStatus.ACTIVE ] return resource
def _setup_unset_enrollment_site(self): site_dao = SiteDao() organization_dao = OrganizationDao() org_2 = organization_dao.insert( Organization(externalId='ORG_2', displayName='Organization 2', hpoId=PITT_HPO_ID)) site_dao.insert( Site(siteName='not enrolling site', googleGroup='not_enrolling_dot_com', organizationId=org_2.organizationId, enrollingStatus=EnrollingStatus.UNSET, siteStatus=SiteStatus.INACTIVE))
def add_missing_hpo_from_site(self, session, participant_id, site_id): if site_id is None: raise BadRequest('No site ID given for auto-pairing participant.') site = SiteDao().get_with_session(session, site_id) if site is None: raise BadRequest('Invalid siteId reference %r.' % site_id) participant = self.get_for_update(session, participant_id) if participant is None: raise BadRequest('No participant %r for HPO ID udpate.' % participant_id) if participant.siteId == site.siteId: return participant.hpoId = site.hpoId participant.organizationId = site.organizationId participant.siteId = site.siteId participant.providerLink = make_primary_provider_link_for_id( site.hpoId) if participant.participantSummary is None: raise RuntimeError('No ParticipantSummary available for P%d.' % participant_id) participant.participantSummary.hpoId = site.hpoId participant.lastModified = clock.CLOCK.now() # Update the version and add history row self._do_update(session, participant, participant)
def test_overwrite_existing_pairing(self): participant_id = 99 created = self.dao.insert( Participant(participantId=participant_id, biobankId=2, hpoId=self._test_db.hpo_id, providerLink=make_primary_provider_link_for_id( self._test_db.hpo_id))) self.participant_summary_dao.insert(self.participant_summary(created)) self.assertEquals(created.hpoId, self._test_db.hpo_id) # sanity check other_hpo = HPODao().insert( HPO(hpoId=PITT_HPO_ID + 1, name='DIFFERENT_HPO')) other_site = SiteDao().insert( Site(hpoId=other_hpo.hpoId, siteName='Arbitrary Site', googleGroup='*****@*****.**')) with self.dao.session() as session: self.dao.add_missing_hpo_from_site(session, participant_id, other_site.siteId) # Original Participant + summary is affected. refetched = self.dao.get(participant_id) self.assertEquals(refetched.hpoId, other_hpo.hpoId) self.assertEquals(refetched.providerLink, make_primary_provider_link_for_id(other_hpo.hpoId)) self.assertEquals( self.participant_summary_dao.get(participant_id).hpoId, other_hpo.hpoId)
def setUp(self, **kwargs): super(MetricsEhrApiTestBase, self).setUp(use_mysql=True, **kwargs) self.dao = ParticipantDao() self.ps_dao = ParticipantSummaryDao() self.ehr_receipt_dao = EhrReceiptDao() self.ps = ParticipantSummary() self.calendar_dao = CalendarDao() self.site_dao = SiteDao() self.hpo_dao = HPODao() self.org_dao = OrganizationDao() self.hpo_test = self._make_hpo(hpoId=TEST_HPO_ID, name=TEST_HPO_NAME, displayName='Test', organizationType=OrganizationType.UNSET) self.hpo_foo = self._make_hpo(hpoId=10, name='FOO', displayName='Foo') self.hpo_bar = self._make_hpo(hpoId=11, name='BAR', displayName='Bar') self.org_foo_a = self._make_org(organizationId=10, externalId='FOO_A', displayName='Foo A', hpoId=self.hpo_foo.hpoId) self.org_bar_a = self._make_org(organizationId=11, externalId='BAR_A', displayName='Bar A', hpoId=self.hpo_bar.hpoId)
def __init__(self): args = parser.parse_args() self.organization_dao = OrganizationDao() self.stub_geocoding = args.stub_geocoding self.ACTIVE = SiteStatus.ACTIVE self.status_exception_list = ['hpo-site-walgreensphoenix'] self.instance = args.instance self.creds_file = args.creds_file self.new_sites_list = [] self.project = None if args.project: self.project = args.project if self.project in ENV_LIST: self.environment = ' ' + self.project.split('-')[-1].upper() else: self.environment = ' ' + ENV_TEST.split('-')[-1].upper() super(SiteImporter, self).__init__('site', SiteDao(), 'siteId', 'googleGroup', [ SITE_ORGANIZATION_ID_COLUMN, SITE_SITE_ID_COLUMN, SITE_SITE_COLUMN, SITE_STATUS_COLUMN + self.environment, ENROLLING_STATUS_COLUMN + self.environment, DIGITAL_SCHEDULING_STATUS_COLUMN + self.environment ])
def get_location_site_id(location_value): if not location_value.startswith(_LOCATION_PREFIX): logging.warn("Invalid location: %s" % location_value) return None google_group = location_value[len(_LOCATION_PREFIX):] site = SiteDao().get_by_google_group(google_group) if not site: logging.warn("Unknown site: %s" % google_group) return None return site.siteId
def __init__(self): super(SiteImporter, self).__init__('site', SiteDao(), 'siteId', 'googleGroup', [ SITE_ORGANIZATION_ID_COLUMN, SITE_SITE_ID_COLUMN, SITE_SITE_COLUMN, SITE_STATUS_COLUMN, ENROLLING_STATUS_COLUMN ]) self.organization_dao = OrganizationDao() args = parser.parse_args() self.geocode_flag = args.geocode_flag self.ACTIVE = SiteStatus.ACTIVE self.status_exception_list = ['hpo-site-walgreensphoenix']
def _cleanup_old_entities(self, session, row_list, dry_run): log_prefix = '(dry run) ' if dry_run else '' self.site_dao = SiteDao() existing_sites = set(site.googleGroup for site in self.site_dao.get_all()) site_group_list_from_sheet = [ str(row[SITE_SITE_ID_COLUMN].lower()) for row in row_list ] sites_to_remove = existing_sites - set(site_group_list_from_sheet) if not sites_to_remove: return site_id_list = [] for site in sites_to_remove: old_site = self.site_dao.get_by_google_group(site) if old_site and old_site.isObsolete != ObsoleteStatus.OBSOLETE: logging.info( log_prefix + 'Deleting old Site no longer in Google sheet: %s', site) site_id_list.append(old_site.siteId) self.deletion_count += 1 elif old_site and old_site.isObsolete == ObsoleteStatus.OBSOLETE: logging.info( 'Not attempting to delete site [%s] with existing obsolete status', old_site.googleGroup) if dry_run or not site_id_list: return # Set obsolete first, in case the sites cannot be deleted below. str_list = ','.join([str(i) for i in site_id_list]) sql = """ UPDATE site SET is_obsolete = 1 WHERE site_id in ({site_id_list})""".format(site_id_list=str_list) session.execute(sql) self.site_dao._invalidate_cache() # Try to delete old sites, this may partially fail if there are still foreign references. self.delete_sql_statement(session, str_list)
def __init__(self, client, use_local_files=None): self._use_local_files = use_local_files self._client = client self._hpos = HPODao().get_all() self._sites = SiteDao().get_all() if not self._sites: raise BadRequest('No sites found; import sites before running generator.') self._now = clock.CLOCK.now() self._consent_questionnaire_id_and_version = None self._setup_data() self._setup_questionnaires() self._min_birth_date = self._now - datetime.timedelta(days=_MAX_PARTICIPANT_AGE * 365) self._max_days_for_birth_date = 365 * (_MAX_PARTICIPANT_AGE - _MIN_PARTICIPANT_AGE)
def _setup_hpos(self, org_dao=None): hpo_dao = HPODao() hpo_dao.insert( HPO(hpoId=UNSET_HPO_ID, name='UNSET', displayName='Unset', organizationType=OrganizationType.UNSET)) hpo_dao.insert( HPO(hpoId=PITT_HPO_ID, name='PITT', displayName='Pittsburgh', organizationType=OrganizationType.HPO)) hpo_dao.insert( HPO(hpoId=AZ_HPO_ID, name='AZ_TUCSON', displayName='Arizona', organizationType=OrganizationType.HPO)) self.hpo_id = PITT_HPO_ID org_dao = OrganizationDao() org_dao.insert( Organization(organizationId=AZ_ORG_ID, externalId='AZ_TUCSON_BANNER_HEALTH', displayName='Banner Health', hpoId=AZ_HPO_ID)) created_org = org_dao.insert( Organization(organizationId=PITT_ORG_ID, externalId='PITT_BANNER_HEALTH', displayName='PITT display Banner Health', hpoId=PITT_HPO_ID)) self.organization_id = created_org.organizationId site_dao = SiteDao() created_site = site_dao.insert( Site(siteName='Monroeville Urgent Care Center', googleGroup='hpo-site-monroeville', mayolinkClientNumber=7035769, organizationId=PITT_ORG_ID, hpoId=PITT_HPO_ID)) self.site_id = created_site.siteId site_dao.insert( Site(siteName='Phoenix Urgent Care Center', googleGroup='hpo-site-bannerphoenix', mayolinkClientNumber=7035770, organizationId=PITT_ORG_ID, hpoId=PITT_HPO_ID)) site_dao.insert( Site(siteName='Phoenix clinic', googleGroup='hpo-site-clinic-phoenix', mayolinkClientNumber=7035770, organizationId=AZ_ORG_ID, hpoId=AZ_HPO_ID))
def _to_handling_info(self, username, site_id): if not username and not site_id: return None info = _FhirBiobankOrderHandlingInfo() if site_id: site = SiteDao().get(site_id) info.site = Identifier() info.site.system = SITE_ID_SYSTEM info.site.value = site.googleGroup if username: info.author = Identifier() info.author.system = HEALTHPRO_USERNAME_SYSTEM info.author.value = username return info
def _setup_hpos(self): hpo_dao = HPODao() hpo_dao.insert(HPO(hpoId=UNSET_HPO_ID, name='UNSET')) hpo_dao.insert( HPO(hpoId=PITT_HPO_ID, name='PITT', organizationType=OrganizationType.HPO)) self.hpo_id = PITT_HPO_ID site_dao = SiteDao() created_site = site_dao.insert( Site(siteName='Monroeville Urgent Care Center', googleGroup='hpo-site-monroeville', consortiumName='Pittsburgh', mayolinkClientNumber=7035769, hpoId=PITT_HPO_ID)) self.site_id = created_site.siteId site_dao.insert( Site(siteName='Phoenix Urgent Care Center', googleGroup='hpo-site-bannerphoenix', consortiumName='Pittsburgh', mayolinkClientNumber=7035770, hpoId=PITT_HPO_ID))
def setUp(self): super(HierarchyContentApiTest, self).setUp(with_data=False) hpo_dao = HPODao() hpo_dao.insert( HPO(hpoId=UNSET_HPO_ID, name='UNSET', displayName='Unset', organizationType=OrganizationType.UNSET, resourceId='h123456')) hpo_dao.insert( HPO(hpoId=PITT_HPO_ID, name='PITT', displayName='Pittsburgh', organizationType=OrganizationType.HPO, resourceId='h123457')) hpo_dao.insert( HPO(hpoId=AZ_HPO_ID, name='AZ_TUCSON', displayName='Arizona', organizationType=OrganizationType.HPO, resourceId='h123458')) self.site_dao = SiteDao() self.org_dao = OrganizationDao()
def _parse_handling_info(self, handling_info): site_id = None username = None if handling_info.site: if handling_info.site.system != SITE_ID_SYSTEM: raise BadRequest('Invalid site system: %s' % handling_info.site.system) site = SiteDao().get_by_google_group(handling_info.site.value) if not site: raise BadRequest('Unrecognized site: %s' % handling_info.site.value) site_id = site.siteId if handling_info.author: if handling_info.author.system != HEALTHPRO_USERNAME_SYSTEM: raise BadRequest('Invalid author system: %s' % handling_info.author.system) username = handling_info.author.value return username, site_id
def _setup_data(self): organization_dao = OrganizationDao() site_dao = SiteDao() org_1 = organization_dao.insert( Organization(externalId='ORG_1', displayName='Organization 1', hpoId=PITT_HPO_ID, resourceId='o123456')) organization_dao.insert( Organization(externalId='AARDVARK_ORG', displayName='Aardvarks Rock', hpoId=PITT_HPO_ID, resourceId='o123457')) site_dao.insert( Site(siteName='Site 1', googleGroup='hpo-site-1', mayolinkClientNumber=123456, organizationId=org_1.organizationId, siteStatus=SiteStatus.ACTIVE, enrollingStatus=EnrollingStatus.ACTIVE, launchDate=datetime.datetime(2016, 1, 1), notes='notes', latitude=12.1, longitude=13.1, directions='directions', physicalLocationName='locationName', address1='address1', address2='address2', city='Austin', state='TX', zipCode='78751', phoneNumber='555-555-5555', adminEmails='[email protected], [email protected]', link='http://www.example.com')) site_dao.insert( Site(siteName='Zebras Rock', googleGroup='aaaaaaa', organizationId=org_1.organizationId, enrollingStatus=EnrollingStatus.INACTIVE, siteStatus=SiteStatus.INACTIVE))
def __init__(self): super(ParticipantDao, self).__init__(Participant) self.hpo_dao = HPODao() self.organization_dao = OrganizationDao() self.site_dao = SiteDao()
class SiteImporter(CsvImporter): def __init__(self): args = parser.parse_args() self.organization_dao = OrganizationDao() self.stub_geocoding = args.stub_geocoding self.ACTIVE = SiteStatus.ACTIVE self.status_exception_list = ['hpo-site-walgreensphoenix'] self.instance = args.instance self.creds_file = args.creds_file self.new_sites_list = [] self.project = None if args.project: self.project = args.project if self.project in ENV_LIST: self.environment = ' ' + self.project.split('-')[-1].upper() else: self.environment = ' ' + ENV_TEST.split('-')[-1].upper() super(SiteImporter, self).__init__('site', SiteDao(), 'siteId', 'googleGroup', [ SITE_ORGANIZATION_ID_COLUMN, SITE_SITE_ID_COLUMN, SITE_SITE_COLUMN, SITE_STATUS_COLUMN + self.environment, ENROLLING_STATUS_COLUMN + self.environment, DIGITAL_SCHEDULING_STATUS_COLUMN + self.environment ]) def run(self, filename, dry_run): super(SiteImporter, self).run(filename, dry_run) insert_participants = False if not dry_run: if self.environment: current_env = ENV_STABLE if self.environment.strip() == 'STABLE' and len( self.new_sites_list) > 0: from googleapiclient.discovery import build logging.info( 'Starting reboot of app instances to insert new test participants' ) service = build('appengine', 'v1', cache_discovery=False) request = service.apps().services().versions().list( appsId=current_env, servicesId='default') versions = request.execute() for version in versions['versions']: if version['servingStatus'] == 'SERVING': _id = version['id'] request = service.apps().services().versions( ).instances().list(servicesId='default', versionsId=_id, appsId=current_env) instances = request.execute() try: for instance in instances['instances']: sha = instance['name'].split('/')[-1] delete_instance = service.apps().services( ).versions().instances().delete( appsId=current_env, servicesId='default', versionsId=_id, instancesId=sha) response = delete_instance.execute() if response['done']: insert_participants = True logging.info( 'Reboot of instance: %s in stable complete.', instance['name']) else: logging.warn( 'Not able to reboot instance on server, Error: %s', response) except KeyError: logging.warn('No running instance for %s', version['name']) if insert_participants: logging.info('Starting import of test participants.') self._insert_new_participants(self.new_sites_list) def delete_sql_statement(self, session, str_list): sql = """ DELETE FROM site WHERE site_id IN ({str_list}) AND NOT EXISTS( SELECT * FROM participant WHERE site_id = site.site_id) AND NOT EXISTS( SELECT * FROM participant_summary WHERE site_id = site.site_id OR physical_measurements_finalized_site_id = site.site_id OR physical_measurements_created_site_id = site.site_id OR biospecimen_source_site_id = site.site_id OR biospecimen_collected_site_id = site.site_id OR biospecimen_processed_site_id = site.site_id OR biospecimen_finalized_site_id = site.site_id ) AND NOT EXISTS( SELECT * FROM participant_history WHERE site_id = site.site_id) AND NOT EXISTS( SELECT * FROM physical_measurements WHERE created_site_id = site.site_id OR finalized_site_id = site.site_id) AND NOT EXISTS( SELECT * FROM biobank_order WHERE finalized_site_id = site.site_id OR source_site_id = site.site_id OR collected_site_id = site.site_id OR processed_site_id = site.site_id ) """.format(str_list=str_list) session.execute(sql) def _cleanup_old_entities(self, session, row_list, dry_run): log_prefix = '(dry run) ' if dry_run else '' self.site_dao = SiteDao() existing_sites = set(site.googleGroup for site in self.site_dao.get_all()) site_group_list_from_sheet = [ str(row[SITE_SITE_ID_COLUMN].lower()) for row in row_list ] sites_to_remove = existing_sites - set(site_group_list_from_sheet) if sites_to_remove: site_id_list = [] for site in sites_to_remove: logging.info( log_prefix + 'Deleting old Site no longer in Google sheet: %s', site) old_site = self.site_dao.get_by_google_group(site) if old_site and old_site.isObsolete != ObsoleteStatus.OBSOLETE: site_id_list.append(old_site.siteId) self.deletion_count += 1 elif old_site and old_site.isObsolete == ObsoleteStatus.OBSOLETE: logging.info( 'Not attempting to delete site [%s] with existing obsolete status', old_site.googleGroup) if site_id_list and not dry_run: str_list = ','.join([str(i) for i in site_id_list]) logging.info(log_prefix + 'Marking old site as obsolete : %s', old_site) sql = """ UPDATE site SET is_obsolete = 1 WHERE site_id in ({site_id_list})""".format(site_id_list=str_list) session.execute(sql) self.site_dao._invalidate_cache() # Try to delete old sites. self.delete_sql_statement(session, str_list) def _insert_new_participants(self, entity): num_participants = 0 participants = { 'zip_code': '20001', 'date_of_birth': '1933-3-3', 'gender_identity': 'GenderIdentity_Woman', 'withdrawalStatus': 'NOT_WITHDRAWN', 'suspensionStatus': 'NOT_SUSPENDED' } client = Client('rdr/v1', False, self.creds_file, self.instance) client_log.setLevel(logging.WARN) questionnaire_to_questions, consent_questionnaire_id_and_version = _setup_questionnaires( client) consent_questions = questionnaire_to_questions[ consent_questionnaire_id_and_version] for site in entity: for participant, v in enumerate(range(1, 21), 1): num_participants += 1 participant = participants participant.update( {'last_name': site.googleGroup.split('-')[-1]}) participant.update({'first_name': 'Participant {}'.format(v)}) participant.update({'site': site.googleGroup}) import_participant(participant, client, consent_questionnaire_id_and_version, questionnaire_to_questions, consent_questions, num_participants) logging.info('%d participants imported.' % num_participants) def _entity_from_row(self, row): google_group = row[SITE_SITE_ID_COLUMN].lower() organization = self.organization_dao.get_by_external_id( row[SITE_ORGANIZATION_ID_COLUMN].upper()) if organization is None: logging.warn('Invalid organization ID %s importing site %s', row[SITE_ORGANIZATION_ID_COLUMN].upper(), google_group) self.errors.append( 'Invalid organization ID {} importing site {}'.format( row[SITE_ORGANIZATION_ID_COLUMN].upper(), google_group)) return None launch_date = None launch_date_str = row.get(SITE_LAUNCH_DATE_COLUMN) if launch_date_str: try: launch_date = parse(launch_date_str).date() except ValueError: logging.warn('Invalid launch date %s for site %s', launch_date_str, google_group) self.errors.append('Invalid launch date {} for site {}'.format( launch_date_str, google_group)) return None name = row[SITE_SITE_COLUMN] mayolink_client_number = None mayolink_client_number_str = row.get( SITE_MAYOLINK_CLIENT_NUMBER_COLUMN) if mayolink_client_number_str: try: mayolink_client_number = int(mayolink_client_number_str) except ValueError: logging.warn('Invalid Mayolink Client # %s for site %s', mayolink_client_number_str, google_group) self.errors.append( 'Invalid Mayolink Client # {} for site {}'.format( mayolink_client_number_str, google_group)) return None notes = row.get(SITE_NOTES_COLUMN) notes_es = row.get(SITE_NOTES_COLUMN_ES) try: site_status = SiteStatus(row[SITE_STATUS_COLUMN + self.environment].upper()) except TypeError: logging.warn('Invalid site status %s for site %s', row[SITE_STATUS_COLUMN + self.environment], google_group) self.errors.append('Invalid site status {} for site {}'.format( row[SITE_STATUS_COLUMN + self.environment], google_group)) return None try: enrolling_status = EnrollingStatus(row[ENROLLING_STATUS_COLUMN + self.environment].upper()) except TypeError: logging.warn('Invalid enrollment site status %s for site %s', row[ENROLLING_STATUS_COLUMN + self.environment], google_group) self.errors.append( 'Invalid enrollment site status {} for site {}'.format( row[ENROLLING_STATUS_COLUMN + self.environment], google_group)) directions = row.get(SITE_DIRECTIONS_COLUMN) physical_location_name = row.get(SITE_PHYSICAL_LOCATION_NAME_COLUMN) address_1 = row.get(SITE_ADDRESS_1_COLUMN) address_2 = row.get(SITE_ADDRESS_2_COLUMN) city = row.get(SITE_CITY_COLUMN) state = row.get(SITE_STATE_COLUMN) zip_code = row.get(SITE_ZIP_COLUMN) phone = row.get(SITE_PHONE_COLUMN) admin_email_addresses = row.get(SITE_ADMIN_EMAIL_ADDRESSES_COLUMN) link = row.get(SITE_LINK_COLUMN) digital_scheduling_status = DigitalSchedulingStatus( row[DIGITAL_SCHEDULING_STATUS_COLUMN + self.environment].upper()) schedule_instructions = row.get(SCHEDULING_INSTRUCTIONS) schedule_instructions_es = row.get(SCHEDULING_INSTRUCTIONS_ES) return Site(siteName=name, googleGroup=google_group, mayolinkClientNumber=mayolink_client_number, organizationId=organization.organizationId, hpoId=organization.hpoId, siteStatus=site_status, enrollingStatus=enrolling_status, digitalSchedulingStatus=digital_scheduling_status, scheduleInstructions=schedule_instructions, scheduleInstructions_ES=schedule_instructions_es, launchDate=launch_date, notes=notes, notes_ES=notes_es, directions=directions, physicalLocationName=physical_location_name, address1=address_1, address2=address_2, city=city, state=state, zipCode=zip_code, phoneNumber=phone, adminEmails=admin_email_addresses, link=link) def _update_entity(self, entity, existing_entity, session, dry_run): self._populate_lat_lng_and_time_zone(entity, existing_entity) if entity.siteStatus == self.ACTIVE and (entity.latitude == None or entity.longitude == None): self.errors.append( 'Skipped active site without geocoding: {}'.format( entity.googleGroup)) return None, True return super(SiteImporter, self)._update_entity(entity, existing_entity, session, dry_run) def _insert_entity(self, entity, existing_map, session, dry_run): self._populate_lat_lng_and_time_zone(entity, None) if entity.siteStatus == self.ACTIVE and (entity.latitude == None or entity.longitude == None): self.errors.append( 'Skipped active site without geocoding: {}'.format( entity.googleGroup)) return False self.new_sites_list.append(entity) super(SiteImporter, self)._insert_entity(entity, existing_map, session, dry_run) def _populate_lat_lng_and_time_zone(self, site, existing_site): if site.address1 and site.city and site.state: if existing_site: if (existing_site.address1 == site.address1 and existing_site.city == site.city and existing_site.state == site.state and existing_site.latitude is not None and existing_site.longitude is not None and existing_site.timeZoneId is not None): # Address didn't change, use the existing lat/lng and time zone. site.latitude = existing_site.latitude site.longitude = existing_site.longitude site.timeZoneId = existing_site.timeZoneId return if self.stub_geocoding: # Set dummy latitude and longitude when importing sites locally / on a CircleCI box. site.latitude = 32.176 site.longitude = -110.93 site.timeZoneId = 'America/Phoenix' else: latitude, longitude = self._get_lat_long_for_site( site.address1, site.city, site.state) site.latitude = latitude site.longitude = longitude if latitude and longitude: site.timeZoneId = self._get_time_zone(latitude, longitude) else: if site.googleGroup not in self.status_exception_list: if site.siteStatus == self.ACTIVE: self.errors.append( 'Active site must have valid address. Site: {}, Group: {}' .format(site.siteName, site.googleGroup)) def _get_lat_long_for_site(self, address_1, city, state): self.full_address = address_1 + ' ' + city + ' ' + state try: self.api_key = os.environ.get('API_KEY') self.gmaps = googlemaps.Client(key=self.api_key) try: geocode_result = self.gmaps.geocode(address_1 + '' + city + ' ' + state)[0] except IndexError: self.errors.append( 'Bad address for {}, could not geocode.'.format( self.full_address)) return None, None if geocode_result: geometry = geocode_result.get('geometry') if geometry: location = geometry.get('location') if location: latitude = location.get('lat') longitude = location.get('lng') return latitude, longitude else: logging.warn('Can not find lat/long for %s', self.full_address) self.errors.append('Can not find lat/long for {}'.format( self.full_address)) return None, None else: logging.warn('Geocode results failed for %s.', self.full_address) self.errors.append('Geocode results failed for {}'.format( self.full_address)) return None, None except ValueError as e: logging.exception('Invalid geocode key: %s. ERROR: %s', self.api_key, e) self.errors.append('Invalid geocode key: {}. ERROR: {}'.format( self.api_key, e)) return None, None except IndexError as e: logging.exception( 'Geocoding failure Check that address is correct. ERROR: %s', e) self.errors.append( 'Geocoding failured Check that address is correct. ERROR: {}'. format(self.api_key, e)) return None, None def _get_time_zone(self, latitude, longitude): time_zone = self.gmaps.timezone(location=(latitude, longitude)) if time_zone['status'] == 'OK': time_zone_id = time_zone['timeZoneId'] return time_zone_id else: logging.info('can not retrieve time zone from %s', self.full_address) self.errors.append('Can not retrieve time zone from {}'.format( self.full_address)) return None
def setUp(self): super(SiteDaoTest, self).setUp() self.site_dao = SiteDao()
def setUp(self): super(SiteDaoTest, self).setUp() self.site_dao = SiteDao() self.participant_dao = ParticipantDao() self.ps_dao = ParticipantSummaryDao() self.ps_history = ParticipantHistoryDao()
class SyncConsentFilesTest(CloudStorageSqlTestBase, NdbTestBase): """Tests behavior of sync_consent_files """ def setUp(self, **kwargs): super(SyncConsentFilesTest, self).setUp(use_mysql=True, **kwargs) NdbTestBase.doSetUp(self) TestBase.setup_fake(self) self.org_dao = OrganizationDao() self.site_dao = SiteDao() self.participant_dao = ParticipantDao() self.summary_dao = ParticipantSummaryDao() def tearDown(self): super(SyncConsentFilesTest, self).tearDown() def _create_org(self, id_): org = Organization(organizationId=id_, externalId=id_, displayName=id_, hpoId=UNSET_HPO_ID) self.org_dao.insert(org) return org def _create_site(self, id_, google_group): site = Site(siteId=id_, siteName=id_, googleGroup=google_group) self.site_dao.insert(site) return site def _create_participant(self, id_, org_id, site_id, consents=False, ghost=None, email=None, null_email=False): participant = Participant(participantId=id_, biobankId=id_, organizationId=org_id, siteId=site_id, isGhostId=ghost) self.participant_dao.insert(participant) summary = self.participant_summary(participant) if consents: summary.consentForElectronicHealthRecords = 1 summary.consentForStudyEnrollment = 1 if email: summary.email = email if null_email: summary.email = None self.summary_dao.insert(summary) return participant def test_iter_participants_data(self): """should list consenting participants """ org1 = self._create_org(1) org2 = self._create_org(2) site1 = self._create_site(1001, 'group1') site2 = self._create_site(1002, 'group2') self._create_participant(1, org1.organizationId, site1.siteId, consents=True, null_email=True) self._create_participant(2, org2.organizationId, site2.siteId) self._create_participant(3, org1.organizationId, None, consents=True, ghost=False) self._create_participant(4, org1.organizationId, None, consents=True, ghost=True) self._create_participant(5, org1.organizationId, None, consents=True, email='*****@*****.**') participant_data_list = list( sync_consent_files._iter_participants_data()) participant_ids = [d.participant_id for d in participant_data_list] self.assertEqual(len(participant_ids), 2, "finds correct number of results") self.assertEqual(participant_ids, [1, 3], "finds valid participants") self.assertEqual(participant_data_list[0].google_group, 'group1', "Includes google group") self.assertEqual(participant_data_list[1].google_group, None, "allows None for google group") @mock.patch('cloudstorage.listbucket') @mock.patch('cloudstorage.copy2') def test_cloudstorage_copy_objects_api_calls(self, mock_copy2, mock_listbucket): """Makes the proper google cloudstorage API calls """ mock_listbucket.return_value = [ cloudstorage.common.GCSFileStat('/fake_bucket1/prefix1/foo', 0, 'x', 0), cloudstorage.common.GCSFileStat('/fake_bucket1/prefix1/bar', 0, 'x', 0), ] # with trailing slashes sync_consent_files.cloudstorage_copy_objects('/fake_bucket1/prefix1/', '/fake_bucket2/prefix2/') mock_copy2.assert_has_calls([ mock.call('/fake_bucket1/prefix1/foo', '/fake_bucket2/prefix2/foo'), mock.call('/fake_bucket1/prefix1/bar', '/fake_bucket2/prefix2/bar'), ]) # without trailing slashes sync_consent_files.cloudstorage_copy_objects('/fake_bucket1/prefix1', '/fake_bucket2/prefix2') mock_copy2.assert_has_calls([ mock.call('/fake_bucket1/prefix1/foo', '/fake_bucket2/prefix2/foo'), mock.call('/fake_bucket1/prefix1/bar', '/fake_bucket2/prefix2/bar'), ]) @staticmethod def _write_cloud_object(cloud_filename, contents_str): with cloudstorage.cloudstorage_api.open(cloud_filename, mode='w') as cloud_file: cloud_file.write(contents_str.encode('utf-8')) def test_cloudstorage_copy_objects_actual(self): self._write_cloud_object('/fake_bucket1/prefix/x1/foo.txt', 'foo') self._write_cloud_object('/fake_bucket1/prefix/x1/bar.txt', 'bar') self._write_cloud_object('/fake_bucket1/prefix/x1/y1/foo.txt', 'foo') with cloudstorage.cloudstorage_api.open( '/fake_bucket1/prefix/x1/foo.txt', mode='r') as f: self.assertEqual(f.read(), 'foo', 'Wrote to cloud storage') sync_consent_files.cloudstorage_copy_objects( '/fake_bucket1/prefix/x1/', '/fake_bucket2/prefix/z/x1/') self.assertEqual([ file_stat.filename for file_stat in cloudstorage.cloudstorage_api.listbucket( '/fake_bucket2/prefix/z/x1/') ], [ '/fake_bucket2/prefix/z/x1/bar.txt', '/fake_bucket2/prefix/z/x1/foo.txt', '/fake_bucket2/prefix/z/x1/y1/foo.txt', ], "copied all objects") with cloudstorage.cloudstorage_api.open( '/fake_bucket2/prefix/z/x1/foo.txt', mode='r') as f: self.assertEqual(f.read(), 'foo', 'copied contents') @mock.patch('cloudstorage.copy2') def test_cloudstorage_copy_objects_only_new_and_changed(self, copy2): self._write_cloud_object('/fake_bucket1/prefix/x1/foo.txt', 'foo') self._write_cloud_object('/fake_bucket1/prefix/x1/bar.txt', 'bar') self._write_cloud_object('/fake_bucket2/prefix/z/x1/foo.txt', 'foo') self._write_cloud_object('/fake_bucket2/prefix/z/x1/bar.txt', 'baz') sync_consent_files.cloudstorage_copy_objects( '/fake_bucket1/prefix/x1/', '/fake_bucket2/prefix/z/x1/') copy2.assert_called_once_with('/fake_bucket1/prefix/x1/bar.txt', '/fake_bucket2/prefix/z/x1/bar.txt')
class SiteDaoTest(SqlTestBase): def setUp(self): super(SiteDaoTest, self).setUp() self.site_dao = SiteDao() self.participant_dao = ParticipantDao() self.ps_dao = ParticipantSummaryDao() self.ps_history = ParticipantHistoryDao() def test_get_no_sites(self): self.assertIsNone(self.site_dao.get(9999)) self.assertIsNone( self.site_dao.get_by_google_group('*****@*****.**')) def test_insert(self): site = Site(siteName='site', googleGroup='*****@*****.**', mayolinkClientNumber=12345, hpoId=PITT_HPO_ID) created_site = self.site_dao.insert(site) new_site = self.site_dao.get(created_site.siteId) site.siteId = created_site.siteId self.assertEquals(site.asdict(), new_site.asdict()) self.assertEquals( site.asdict(), self.site_dao.get_by_google_group( '*****@*****.**').asdict()) def test_update(self): site = Site(siteName='site', googleGroup='*****@*****.**', mayolinkClientNumber=12345, hpoId=PITT_HPO_ID) created_site = self.site_dao.insert(site) new_site = Site(siteId=created_site.siteId, siteName='site2', googleGroup='*****@*****.**', mayolinkClientNumber=123456, hpoId=UNSET_HPO_ID) self.site_dao.update(new_site) fetched_site = self.site_dao.get(created_site.siteId) self.assertEquals(new_site.asdict(), fetched_site.asdict()) self.assertEquals( new_site.asdict(), self.site_dao.get_by_google_group( '*****@*****.**').asdict()) self.assertIsNone( self.site_dao.get_by_google_group('*****@*****.**')) def test_participant_pairing_updates_on_change(self): TIME = datetime.datetime(2018, 1, 1) TIME2 = datetime.datetime(2018, 1, 2) provider_link = '[{"organization": {"reference": "Organization/AZ_TUCSON"}, "primary": true}]' site = Site(siteName='site', googleGroup='*****@*****.**', mayolinkClientNumber=12345, hpoId=PITT_HPO_ID, organizationId=PITT_ORG_ID) created_site = self.site_dao.insert(site) with FakeClock(TIME): p = Participant(participantId=1, biobankId=2, siteId=created_site.siteId) self.participant_dao.insert(p) fetch_p = self.participant_dao.get(p.participantId) updated_p = self.participant_dao.get(fetch_p.participantId) p_summary = self.ps_dao.insert(self.participant_summary(updated_p)) with FakeClock(TIME2): update_site_parent = Site(siteId=created_site.siteId, siteName='site2', googleGroup='*****@*****.**', mayolinkClientNumber=123456, hpoId=AZ_HPO_ID, organizationId=AZ_ORG_ID) self.site_dao.update(update_site_parent) updated_p = self.participant_dao.get(fetch_p.participantId) ps = self.ps_dao.get(p_summary.participantId) ph = self.ps_history.get([updated_p.participantId, 1]) self.assertEquals(update_site_parent.hpoId, updated_p.hpoId) self.assertEquals(update_site_parent.organizationId, updated_p.organizationId) self.assertEquals(ps.organizationId, update_site_parent.organizationId) self.assertEquals(ps.hpoId, update_site_parent.hpoId) self.assertEquals(ps.organizationId, update_site_parent.organizationId) self.assertEquals(ph.organizationId, update_site_parent.organizationId) self.assertEquals(updated_p.providerLink, provider_link) self.assertEquals(ps.lastModified, TIME2)
class ParticipantSummaryDao(UpdatableDao): def __init__(self): super(ParticipantSummaryDao, self).__init__(ParticipantSummary, order_by_ending=_ORDER_BY_ENDING) self.hpo_dao = HPODao() self.code_dao = CodeDao() self.site_dao = SiteDao() self.organization_dao = OrganizationDao() def get_id(self, obj): return obj.participantId def _validate_update(self, session, obj, existing_obj): # pylint: disable=unused-argument """Participant summaries don't have a version value; drop it from validation logic.""" if not existing_obj: raise NotFound('%s with id %s does not exist' % (self.model_type.__name__, id)) def _has_withdrawn_filter(self, query): for field_filter in query.field_filters: if (field_filter.field_name == 'withdrawalStatus' and field_filter.value == WithdrawalStatus.NO_USE): return True if field_filter.field_name == 'withdrawalTime' and field_filter.value is not None: return True return False def _get_non_withdrawn_filter_field(self, query): """Returns the first field referenced in query filters which isn't in WITHDRAWN_PARTICIPANT_FIELDS.""" for field_filter in query.field_filters: if not field_filter.field_name in WITHDRAWN_PARTICIPANT_FIELDS: return field_filter.field_name return None def _initialize_query(self, session, query_def): non_withdrawn_field = self._get_non_withdrawn_filter_field(query_def) if self._has_withdrawn_filter(query_def): if non_withdrawn_field: raise BadRequest( "Can't query on %s for withdrawn participants" % non_withdrawn_field) # When querying for withdrawn participants, ensure that the only fields being filtered on or # ordered by are in WITHDRAWN_PARTICIPANT_FIELDS. return super(ParticipantSummaryDao, self)._initialize_query(session, query_def) else: query = super(ParticipantSummaryDao, self)._initialize_query(session, query_def) if non_withdrawn_field: # When querying on fields that aren't available for withdrawn participants, # ensure that we only return participants # who have not withdrawn or withdrew in the past 48 hours. withdrawn_visible_start = clock.CLOCK.now( ) - WITHDRAWN_PARTICIPANT_VISIBILITY_TIME return query.filter( or_( ParticipantSummary.withdrawalStatus != WithdrawalStatus.NO_USE, ParticipantSummary.withdrawalTime >= withdrawn_visible_start)) else: # When querying on fields that are available for withdrawn participants, return everybody; # withdrawn participants will have all but WITHDRAWN_PARTICIPANT_FIELDS cleared out 48 # hours after withdrawing. return query def _get_order_by_ending(self, query): if self._has_withdrawn_filter(query): return _WITHDRAWN_ORDER_BY_ENDING return self.order_by_ending def _add_order_by(self, query, order_by, field_names, fields): if order_by.field_name in _CODE_FILTER_FIELDS: return super(ParticipantSummaryDao, self)._add_order_by( query, OrderBy(order_by.field_name + 'Id', order_by.ascending), field_names, fields) return super(ParticipantSummaryDao, self)._add_order_by(query, order_by, field_names, fields) def make_query_filter(self, field_name, value): """Handle HPO and code values when parsing filter values.""" if field_name == 'biobankId': value = from_client_biobank_id(value, log_exception=True) if field_name == 'hpoId' or field_name == 'awardee': hpo = self.hpo_dao.get_by_name(value) if not hpo: raise BadRequest('No HPO found with name %s' % value) if field_name == 'awardee': field_name = 'hpoId' return super(ParticipantSummaryDao, self).make_query_filter(field_name, hpo.hpoId) if field_name == 'organization': organization = self.organization_dao.get_by_external_id(value) if not organization: raise BadRequest('No organization found with name %s' % value) return super(ParticipantSummaryDao, self).make_query_filter(field_name + 'Id', organization.organizationId) if field_name in _SITE_FIELDS: if value == UNSET: return super(ParticipantSummaryDao, self).make_query_filter(field_name + 'Id', None) site = self.site_dao.get_by_google_group(value) if not site: raise BadRequest('No site found with google group %s' % value) return super(ParticipantSummaryDao, self).make_query_filter(field_name + 'Id', site.siteId) if field_name in _CODE_FILTER_FIELDS: if value == UNSET: return super(ParticipantSummaryDao, self).make_query_filter(field_name + 'Id', None) # Note: we do not at present support querying for UNMAPPED code values. code = self.code_dao.get_code(PPI_SYSTEM, value) if not code: raise BadRequest('No code found: %s' % value) return super(ParticipantSummaryDao, self).make_query_filter(field_name + 'Id', code.codeId) return super(ParticipantSummaryDao, self).make_query_filter(field_name, value) def update_from_biobank_stored_samples(self, participant_id=None): """Rewrites sample-related summary data. Call this after updating BiobankStoredSamples. If participant_id is provided, only that participant will have their summary updated.""" now = clock.CLOCK.now() sample_sql, sample_params = _get_sample_sql_and_params(now) baseline_tests_sql, baseline_tests_params = _get_baseline_sql_and_params( ) dna_tests_sql, dna_tests_params = _get_dna_isolates_sql_and_params() sample_status_time_sql = _get_sample_status_time_sql_and_params() sample_status_time_params = {} counts_sql = """ UPDATE participant_summary SET num_baseline_samples_arrived = {baseline_tests_sql}, samples_to_isolate_dna = {dna_tests_sql}, last_modified = :now WHERE num_baseline_samples_arrived != {baseline_tests_sql} OR samples_to_isolate_dna != {dna_tests_sql} """.format(baseline_tests_sql=baseline_tests_sql, dna_tests_sql=dna_tests_sql) counts_params = {'now': now} counts_params.update(baseline_tests_params) counts_params.update(dna_tests_params) enrollment_status_sql = _ENROLLMENT_STATUS_SQL enrollment_status_params = { 'submitted': int(QuestionnaireStatus.SUBMITTED), 'unset': int(QuestionnaireStatus.UNSET), 'num_baseline_ppi_modules': self._get_num_baseline_ppi_modules(), 'completed': int(PhysicalMeasurementsStatus.COMPLETED), 'received': int(SampleStatus.RECEIVED), 'full_participant': int(EnrollmentStatus.FULL_PARTICIPANT), 'member': int(EnrollmentStatus.MEMBER), 'interested': int(EnrollmentStatus.INTERESTED), 'now': now } # If participant_id is provided, add the participant ID filter to all update statements. if participant_id: sample_sql += ' AND participant_id = :participant_id' sample_params['participant_id'] = participant_id counts_sql += ' AND participant_id = :participant_id' counts_params['participant_id'] = participant_id enrollment_status_sql += ' AND participant_id = :participant_id' enrollment_status_params['participant_id'] = participant_id sample_status_time_sql += ' AND a.participant_id = :participant_id' sample_status_time_params['participant_id'] = participant_id sample_sql = replace_null_safe_equals(sample_sql) counts_sql = replace_null_safe_equals(counts_sql) with self.session() as session: session.execute(sample_sql, sample_params) session.execute(counts_sql, counts_params) session.execute(enrollment_status_sql, enrollment_status_params) # TODO: Change this to the optimized sql in _update_dv_stored_samples() session.execute(sample_status_time_sql, sample_status_time_params) def _get_num_baseline_ppi_modules(self): return len( config.getSettingList(config.BASELINE_PPI_QUESTIONNAIRE_FIELDS)) def update_enrollment_status(self, summary): """Updates the enrollment status field on the provided participant summary to the correct value based on the other fields on it. Called after a questionnaire response or physical measurements are submitted.""" consent = (summary.consentForStudyEnrollment == QuestionnaireStatus.SUBMITTED and summary.consentForElectronicHealthRecords == QuestionnaireStatus.SUBMITTED) or \ (summary.consentForStudyEnrollment == QuestionnaireStatus.SUBMITTED and summary.consentForElectronicHealthRecords is None and summary.consentForDvElectronicHealthRecordsSharing == QuestionnaireStatus.SUBMITTED) enrollment_status = self.calculate_enrollment_status( consent, summary.numCompletedBaselinePPIModules, summary.physicalMeasurementsStatus, summary.samplesToIsolateDNA) summary.enrollmentStatusMemberTime = self.calculate_member_time( consent, summary) summary.enrollmentStatusCoreOrderedSampleTime = self.calculate_core_ordered_sample_time( consent, summary) summary.enrollmentStatusCoreStoredSampleTime = self.calculate_core_stored_sample_time( consent, summary) # Update last modified date if status changes if summary.enrollmentStatus != enrollment_status: summary.lastModified = clock.CLOCK.now() summary.enrollmentStatus = enrollment_status def calculate_enrollment_status(self, consent, num_completed_baseline_ppi_modules, physical_measurements_status, samples_to_isolate_dna): if consent: if (num_completed_baseline_ppi_modules == self._get_num_baseline_ppi_modules() and physical_measurements_status == PhysicalMeasurementsStatus.COMPLETED and samples_to_isolate_dna == SampleStatus.RECEIVED): return EnrollmentStatus.FULL_PARTICIPANT return EnrollmentStatus.MEMBER return EnrollmentStatus.INTERESTED def calculate_member_time(self, consent, participant_summary): if consent and participant_summary.enrollmentStatusMemberTime is not None: return participant_summary.enrollmentStatusMemberTime elif consent: if participant_summary.consentForElectronicHealthRecords is None and \ participant_summary.consentForDvElectronicHealthRecordsSharing == \ QuestionnaireStatus.SUBMITTED: return participant_summary.consentForDvElectronicHealthRecordsSharingTime return participant_summary.consentForElectronicHealthRecordsTime else: return None def calculate_core_stored_sample_time(self, consent, participant_summary): if consent and \ participant_summary.numCompletedBaselinePPIModules == \ self._get_num_baseline_ppi_modules() and \ participant_summary.physicalMeasurementsStatus == PhysicalMeasurementsStatus.COMPLETED and \ participant_summary.samplesToIsolateDNA == SampleStatus.RECEIVED: max_core_sample_time = self.calculate_max_core_sample_time( participant_summary, field_name_prefix='sampleStatus') if max_core_sample_time and participant_summary.enrollmentStatusCoreStoredSampleTime: return participant_summary.enrollmentStatusCoreStoredSampleTime else: return max_core_sample_time else: return None def calculate_core_ordered_sample_time(self, consent, participant_summary): if consent and \ participant_summary.numCompletedBaselinePPIModules == \ self._get_num_baseline_ppi_modules() and \ participant_summary.physicalMeasurementsStatus == PhysicalMeasurementsStatus.COMPLETED: max_core_sample_time = self.calculate_max_core_sample_time( participant_summary, field_name_prefix='sampleOrderStatus') if max_core_sample_time and participant_summary.enrollmentStatusCoreOrderedSampleTime: return participant_summary.enrollmentStatusCoreOrderedSampleTime else: return max_core_sample_time else: return None def calculate_max_core_sample_time(self, participant_summary, field_name_prefix='sampleStatus'): keys = [ field_name_prefix + '%sTime' % test for test in config.getSettingList(config.DNA_SAMPLE_TEST_CODES) ] sample_time_list = \ [v for k, v in participant_summary if k in keys and v is not None] sample_time = min(sample_time_list) if sample_time_list else None if sample_time is not None: return max([ sample_time, participant_summary.enrollmentStatusMemberTime, participant_summary.questionnaireOnTheBasicsTime, participant_summary.questionnaireOnLifestyleTime, participant_summary.questionnaireOnOverallHealthTime, participant_summary.physicalMeasurementsFinalizedTime ]) else: return None def calculate_distinct_visits(self, pid, finalized_time, id_, amendment=False): """ Participants may get PM or biobank samples on same day. This should be considered as a single visit in terms of program payment to participant. return Boolean: true if there has not been an order on same date.""" from dao.biobank_order_dao import BiobankOrderDao from dao.physical_measurements_dao import PhysicalMeasurementsDao day_has_order, day_has_measurement = False, False existing_orders = BiobankOrderDao().get_biobank_orders_for_participant( pid) ordered_samples = BiobankOrderDao( ).get_ordered_samples_for_participant(pid) existing_measurements = PhysicalMeasurementsDao( ).get_measuremnets_for_participant(pid) order_id_to_finalized_date = { sample.biobankOrderId: sample.finalized.date() for sample in ordered_samples if sample.finalized } if existing_orders and finalized_time: for order in existing_orders: order_finalized_date = order_id_to_finalized_date.get( order.biobankOrderId) if order_finalized_date == finalized_time.date() and order.biobankOrderId != id_ and \ order.orderStatus != BiobankOrderStatus.CANCELLED: day_has_order = True elif order.biobankOrderId == id_ and order.orderStatus == BiobankOrderStatus.AMENDED: day_has_order = True elif not finalized_time and amendment: day_has_order = True if existing_measurements and finalized_time: for measurement in existing_measurements: if not measurement.finalized: continue if measurement.finalized.date() == finalized_time.date() and measurement.physicalMeasurementsId\ != id_: day_has_measurement = True is_distinct_visit = not (day_has_order or day_has_measurement) return is_distinct_visit def to_client_json(self, model): result = model.asdict() # Participants that withdrew more than 48 hours ago should have fields other than # WITHDRAWN_PARTICIPANT_FIELDS cleared. if (model.withdrawalStatus == WithdrawalStatus.NO_USE and (model.withdrawalTime is None or model.withdrawalTime < clock.CLOCK.now() - WITHDRAWN_PARTICIPANT_VISIBILITY_TIME)): result = {k: result.get(k) for k in WITHDRAWN_PARTICIPANT_FIELDS} elif model.withdrawalStatus != WithdrawalStatus.NO_USE and \ model.suspensionStatus == SuspensionStatus.NO_CONTACT: for i in SUSPENDED_PARTICIPANT_FIELDS: result[i] = UNSET result['participantId'] = to_client_participant_id(model.participantId) biobank_id = result.get('biobankId') if biobank_id: result['biobankId'] = to_client_biobank_id(biobank_id) date_of_birth = result.get('dateOfBirth') if date_of_birth: result['ageRange'] = get_bucketed_age(date_of_birth, clock.CLOCK.now()) else: result['ageRange'] = UNSET if result.get('primaryLanguage') is None: result['primaryLanguage'] = UNSET if 'organizationId' in result: result['organization'] = result['organizationId'] del result['organizationId'] format_json_org(result, self.organization_dao, 'organization') format_json_hpo(result, self.hpo_dao, 'hpoId') result['awardee'] = result['hpoId'] _initialize_field_type_sets() for fieldname in _DATE_FIELDS: format_json_date(result, fieldname) for fieldname in _CODE_FIELDS: format_json_code(result, self.code_dao, fieldname) for fieldname in _ENUM_FIELDS: format_json_enum(result, fieldname) for fieldname in _SITE_FIELDS: format_json_site(result, self.site_dao, fieldname) if (model.withdrawalStatus == WithdrawalStatus.NO_USE or model.suspensionStatus == SuspensionStatus.NO_CONTACT): result['recontactMethod'] = 'NO_CONTACT' # Strip None values. result = {k: v for k, v in result.iteritems() if v is not None} return result def _decode_token(self, query_def, fields): """ If token exists in participant_summary api, decode and use lastModified to add a buffer of 60 seconds. This ensures when a _sync link is used no one is missed. This will return at a minimum, the last participant and any more that have been modified in the previous 60 seconds. Duplicate participants returned should be handled on the client side.""" decoded_vals = super(ParticipantSummaryDao, self)._decode_token(query_def, fields) if query_def.order_by and (query_def.order_by.field_name == 'lastModified' and query_def.always_return_token is True and query_def.backfill_sync is True): decoded_vals[0] = decoded_vals[0] - datetime.timedelta( seconds=config.LAST_MODIFIED_BUFFER_SECONDS) return decoded_vals @staticmethod def update_ehr_status(summary, update_time): summary.ehrStatus = EhrStatus.PRESENT if not summary.ehrReceiptTime: summary.ehrReceiptTime = update_time summary.ehrUpdateTime = update_time return summary
class SiteDaoTest(SqlTestBase): def setUp(self): super(SiteDaoTest, self).setUp() self.site_dao = SiteDao() def test_get_no_sites(self): self.assertIsNone(self.site_dao.get(9999)) self.assertIsNone( self.site_dao.get_by_google_group('*****@*****.**')) def test_insert(self): site = Site(siteName='site', googleGroup='*****@*****.**', consortiumName='consortium', mayolinkClientNumber=12345, hpoId=PITT_HPO_ID) created_site = self.site_dao.insert(site) new_site = self.site_dao.get(created_site.siteId) site.siteId = created_site.siteId self.assertEquals(site.asdict(), new_site.asdict()) self.assertEquals( site.asdict(), self.site_dao.get_by_google_group( '*****@*****.**').asdict()) def test_update(self): site = Site(siteName='site', googleGroup='*****@*****.**', consortiumName='consortium', mayolinkClientNumber=12345, hpoId=PITT_HPO_ID) created_site = self.site_dao.insert(site) new_site = Site(siteId=created_site.siteId, siteName='site2', googleGroup='*****@*****.**', consortiumName='consortium2', mayolinkClientNumber=123456, hpoId=UNSET_HPO_ID) self.site_dao.update(new_site) fetched_site = self.site_dao.get(created_site.siteId) self.assertEquals(new_site.asdict(), fetched_site.asdict()) self.assertEquals( new_site.asdict(), self.site_dao.get_by_google_group( '*****@*****.**').asdict()) self.assertIsNone( self.site_dao.get_by_google_group('*****@*****.**'))
class ParticipantDao(UpdatableDao): def __init__(self): super(ParticipantDao, self).__init__(Participant) self.hpo_dao = HPODao() self.organization_dao = OrganizationDao() self.site_dao = SiteDao() def get_id(self, obj): return obj.participantId def insert_with_session(self, session, obj): obj.hpoId = self._get_hpo_id(obj) obj.version = 1 obj.signUpTime = clock.CLOCK.now().replace(microsecond=0) obj.lastModified = obj.signUpTime if obj.withdrawalStatus is None: obj.withdrawalStatus = WithdrawalStatus.NOT_WITHDRAWN if obj.suspensionStatus is None: obj.suspensionStatus = SuspensionStatus.NOT_SUSPENDED super(ParticipantDao, self).insert_with_session(session, obj) history = ParticipantHistory() history.fromdict(obj.asdict(), allow_pk=True) session.add(history) return obj def insert(self, obj): if obj.participantId: assert obj.biobankId return super(ParticipantDao, self).insert(obj) assert not obj.biobankId return self._insert_with_random_id(obj, ('participantId', 'biobankId')) def update_ghost_participant(self, session, pid): if not pid: raise Forbidden('Can not update participant without id') participant = self.get_for_update(session, pid) if participant is None: logging.warn('Tried to mark participant with id: [%r] as ghost but participant does not' 'exist. Wrong environment?' % pid) else: participant.isGhostId = 1 participant.dateAddedGhost = clock.CLOCK.now() self._update_history(session, participant, participant) super(ParticipantDao, self)._do_update(session, participant, participant) def _check_if_external_id_exists(self, obj): with self.session() as session: return session.query(Participant).filter_by(externalId=obj.externalId).first() def _update_history(self, session, obj, existing_obj): # Increment the version and add a new history entry. obj.version = existing_obj.version + 1 history = ParticipantHistory() history.fromdict(obj.asdict(), allow_pk=True) session.add(history) def _validate_update(self, session, obj, existing_obj): # Withdrawal and suspension have default values assigned on insert, so they should always have # explicit values in updates. if obj.withdrawalStatus is None: raise BadRequest('missing withdrawal status in update') if obj.suspensionStatus is None: raise BadRequest('missing suspension status in update') if obj.withdrawalReason != WithdrawalReason.UNSET and obj.withdrawalReason is not None and \ obj.withdrawalReasonJustification is None: raise BadRequest('missing withdrawalReasonJustification in update') super(ParticipantDao, self)._validate_update(session, obj, existing_obj) # Once a participant marks their withdrawal status as NO_USE, it can't be changed back. # TODO: Consider the future ability to un-withdraw. if (existing_obj.withdrawalStatus == WithdrawalStatus.NO_USE and obj.withdrawalStatus != WithdrawalStatus.NO_USE): raise Forbidden('Participant %d has withdrawn, cannot unwithdraw' % obj.participantId) def get_for_update(self, session, obj_id): # Fetch the participant summary at the same time as the participant, as we are potentially # updating both. return self.get_with_session(session, obj_id, for_update=True, options=joinedload(Participant.participantSummary)) def _do_update(self, session, obj, existing_obj): """Updates the associated ParticipantSummary, and extracts HPO ID from the provider link or set pairing at another level (site/organization/awardee) with parent/child enforcement.""" obj.lastModified = clock.CLOCK.now() obj.signUpTime = existing_obj.signUpTime obj.biobankId = existing_obj.biobankId obj.withdrawalTime = existing_obj.withdrawalTime obj.suspensionTime = existing_obj.suspensionTime need_new_summary = False if obj.withdrawalStatus != existing_obj.withdrawalStatus: obj.withdrawalTime = (obj.lastModified if obj.withdrawalStatus == WithdrawalStatus.NO_USE else None) obj.withdrawalAuthored = obj.withdrawalAuthored \ if obj.withdrawalStatus == WithdrawalStatus.NO_USE else None need_new_summary = True if obj.suspensionStatus != existing_obj.suspensionStatus: obj.suspensionTime = (obj.lastModified if obj.suspensionStatus == SuspensionStatus.NO_CONTACT else None) need_new_summary = True update_pairing = True if obj.siteId is None and obj.organizationId is None and obj.hpoId is None and \ obj.providerLink == 'null': # Prevent unpairing if /PUT is sent with no pairing levels. update_pairing = False if update_pairing is True: has_id = False if obj.organizationId or obj.siteId or (obj.hpoId >= 0): has_id = True provider_link_unchanged = True if obj.providerLink is not None: if existing_obj.providerLink: provider_link_unchanged = json.loads(obj.providerLink) == \ json.loads(existing_obj.providerLink) else: provider_link_unchanged = False null_provider_link = obj.providerLink == 'null' # site,org,or awardee is sent in request: Get relationships and try to set provider link. if has_id and (provider_link_unchanged or null_provider_link): site, organization, awardee = self.get_pairing_level(obj) obj.organizationId = organization obj.siteId = site obj.hpoId = awardee if awardee is not None and (obj.hpoId != existing_obj.hpoId): # get provider link for hpo_id (awardee) obj.providerLink = make_primary_provider_link_for_id(awardee) need_new_summary = True else: # providerLink has changed # If the provider link changes, update the HPO ID on the participant and its summary. if obj.hpoId is None: obj.hpoId = existing_obj.hpoId new_hpo_id = self._get_hpo_id(obj) if new_hpo_id != existing_obj.hpoId: obj.hpoId = new_hpo_id obj.siteId = None obj.organizationId = None need_new_summary = True # No pairing updates sent, keep existing values. if update_pairing == False: obj.siteId = existing_obj.siteId obj.organizationId = existing_obj.organizationId obj.hpoId = existing_obj.hpoId obj.providerLink = existing_obj.providerLink if need_new_summary and existing_obj.participantSummary: # Copy the existing participant summary, and mutate the fields that # come from participant. summary = existing_obj.participantSummary summary.hpoId = obj.hpoId summary.organizationId = obj.organizationId summary.siteId = obj.siteId summary.withdrawalStatus = obj.withdrawalStatus summary.withdrawalReason = obj.withdrawalReason summary.withdrawalReasonJustification = obj.withdrawalReasonJustification summary.withdrawalTime = obj.withdrawalTime summary.withdrawalAuthored = obj.withdrawalAuthored summary.suspensionStatus = obj.suspensionStatus summary.suspensionTime = obj.suspensionTime summary.lastModified = clock.CLOCK.now() make_transient(summary) make_transient(obj) obj.participantSummary = summary self._update_history(session, obj, existing_obj) super(ParticipantDao, self)._do_update(session, obj, existing_obj) def get_pairing_level(self, obj): organization_id = obj.organizationId site_id = obj.siteId awardee_id = obj.hpoId # TODO: DO WE WANT TO PREVENT PAIRING IF EXISTING SITE HAS PM/BIO. if site_id != UNSET and site_id is not None: site = self.site_dao.get(site_id) if site is None: raise BadRequest('Site with site id %s does not exist.' % site_id) organization_id = site.organizationId awardee_id = site.hpoId return site_id, organization_id, awardee_id elif organization_id != UNSET and organization_id is not None: organization = self.organization_dao.get(organization_id) if organization is None: raise BadRequest('Organization with id %s does not exist.' % organization_id) awardee_id = organization.hpoId return None, organization_id, awardee_id return None, None, awardee_id @staticmethod def create_summary_for_participant(obj): return ParticipantSummary( participantId=obj.participantId, lastModified=obj.lastModified, biobankId=obj.biobankId, signUpTime=obj.signUpTime, hpoId=obj.hpoId, organizationId=obj.organizationId, siteId=obj.siteId, withdrawalStatus=obj.withdrawalStatus, withdrawalReason=obj.withdrawalReason, withdrawalReasonJustification=obj.withdrawalReasonJustification, suspensionStatus=obj.suspensionStatus, enrollmentStatus=EnrollmentStatus.INTERESTED, ehrStatus=EhrStatus.NOT_PRESENT) @staticmethod def _get_hpo_id(obj): hpo_name = _get_hpo_name_from_participant(obj) if hpo_name: hpo = HPODao().get_by_name(hpo_name) if not hpo: raise BadRequest('No HPO found with name %s' % hpo_name) return hpo.hpoId else: return UNSET_HPO_ID def validate_participant_reference(self, session, obj): """Raises BadRequest if an object has a missing or invalid participantId reference, or if the participant has a withdrawal status of NO_USE.""" if obj.participantId is None: raise BadRequest('%s.participantId required.' % obj.__class__.__name__) return self.validate_participant_id(session, obj.participantId) def validate_participant_id(self, session, participant_id): """Raises BadRequest if a participant ID is invalid, or if the participant has a withdrawal status of NO_USE.""" participant = self.get_with_session(session, participant_id) if participant is None: raise BadRequest('Participant with ID %d is not found.' % participant_id) raise_if_withdrawn(participant) return participant def get_biobank_ids_sample(self, session, percentage, batch_size): """Returns biobank ID and signUpTime for a percentage of participants. Used in generating fake biobank samples.""" return (session.query(Participant.biobankId, Participant.signUpTime) .filter(Participant.biobankId % 100 <= percentage * 100) .yield_per(batch_size)) def to_client_json(self, model): client_json = { 'participantId': to_client_participant_id(model.participantId), 'externalId': model.externalId, 'hpoId': model.hpoId, 'awardee': model.hpoId, 'organization': model.organizationId, 'siteId': model.siteId, 'biobankId': to_client_biobank_id(model.biobankId), 'lastModified': model.lastModified.isoformat(), 'signUpTime': model.signUpTime.isoformat(), 'providerLink': json.loads(model.providerLink), 'withdrawalStatus': model.withdrawalStatus, 'withdrawalReason': model.withdrawalReason, 'withdrawalReasonJustification': model.withdrawalReasonJustification, 'withdrawalTime': model.withdrawalTime, 'withdrawalAuthored': model.withdrawalAuthored, 'suspensionStatus': model.suspensionStatus, 'suspensionTime': model.suspensionTime } format_json_hpo(client_json, self.hpo_dao, 'hpoId'), format_json_org(client_json, self.organization_dao, 'organization'), format_json_site(client_json, self.site_dao, 'site'), format_json_enum(client_json, 'withdrawalStatus') format_json_enum(client_json, 'withdrawalReason') format_json_enum(client_json, 'suspensionStatus') format_json_date(client_json, 'withdrawalTime') format_json_date(client_json, 'suspensionTime') client_json['awardee'] = client_json['hpoId'] if 'siteId' in client_json: del client_json['siteId'] return client_json def from_client_json(self, resource_json, id_=None, expected_version=None, client_id=None): parse_json_enum(resource_json, 'withdrawalStatus', WithdrawalStatus) parse_json_enum(resource_json, 'withdrawalReason', WithdrawalReason) parse_json_enum(resource_json, 'suspensionStatus', SuspensionStatus) if 'withdrawalTimeStamp' in resource_json and resource_json['withdrawalTimeStamp'] is not None: try: resource_json['withdrawalTimeStamp'] = datetime.datetime\ .utcfromtimestamp(float(resource_json['withdrawalTimeStamp'])/1000) except (ValueError, TypeError): raise ValueError("Could not parse {} as TIMESTAMP" .format(resource_json['withdrawalTimeStamp'])) # biobankId, lastModified, signUpTime are set by DAO. return Participant( participantId=id_, externalId=resource_json.get('externalId'), version=expected_version, providerLink=json.dumps(resource_json.get('providerLink')), clientId=client_id, withdrawalStatus=resource_json.get('withdrawalStatus'), withdrawalReason=resource_json.get('withdrawalReason'), withdrawalAuthored=resource_json.get('withdrawalTimeStamp'), withdrawalReasonJustification=resource_json.get('withdrawalReasonJustification'), suspensionStatus=resource_json.get('suspensionStatus'), organizationId=get_organization_id_from_external_id(resource_json, self.organization_dao), hpoId=get_awardee_id_from_name(resource_json, self.hpo_dao), siteId=get_site_id_from_google_group(resource_json, self.site_dao)) def add_missing_hpo_from_site(self, session, participant_id, site_id): if site_id is None: raise BadRequest('No site ID given for auto-pairing participant.') site = SiteDao().get_with_session(session, site_id) if site is None: raise BadRequest('Invalid siteId reference %r.' % site_id) participant = self.get_for_update(session, participant_id) if participant is None: raise BadRequest('No participant %r for HPO ID udpate.' % participant_id) if participant.siteId == site.siteId: return participant.hpoId = site.hpoId participant.organizationId = site.organizationId participant.siteId = site.siteId participant.providerLink = make_primary_provider_link_for_id(site.hpoId) if participant.participantSummary is None: raise RuntimeError('No ParticipantSummary available for P%d.' % participant_id) participant.participantSummary.hpoId = site.hpoId participant.lastModified = clock.CLOCK.now() # Update the version and add history row self._do_update(session, participant, participant) def switch_to_test_account(self, session, participant): test_hpo_id = HPODao().get_by_name(TEST_HPO_NAME).hpoId if participant is None: raise BadRequest('No participant %r for HPO ID udpate.') if participant.hpoId == test_hpo_id: return participant.hpoId = test_hpo_id participant.organizationId = None participant.siteId = None # Update the version and add history row self._do_update(session, participant, participant) def handle_integrity_error(self, tried_ids, e, obj): if 'external_id' in e.message: existing_participant = self._check_if_external_id_exists(obj) if existing_participant: return existing_participant return super(ParticipantDao, self).handle_integrity_error(tried_ids, e, obj)
class HierarchyContentApiTest(FlaskTestBase): def setUp(self): super(HierarchyContentApiTest, self).setUp(with_data=False) hpo_dao = HPODao() hpo_dao.insert( HPO(hpoId=UNSET_HPO_ID, name='UNSET', displayName='Unset', organizationType=OrganizationType.UNSET, resourceId='h123456')) hpo_dao.insert( HPO(hpoId=PITT_HPO_ID, name='PITT', displayName='Pittsburgh', organizationType=OrganizationType.HPO, resourceId='h123457')) hpo_dao.insert( HPO(hpoId=AZ_HPO_ID, name='AZ_TUCSON', displayName='Arizona', organizationType=OrganizationType.HPO, resourceId='h123458')) self.site_dao = SiteDao() self.org_dao = OrganizationDao() def test_create_new_hpo(self): self._setup_data() request_json = { "resourceType": "Organization", "id": "a893282c-2717-4a20-b276-d5c9c2c0e51f", 'meta': { 'versionId': '1' }, "extension": [{ "url": "http://all-of-us.org/fhir/sites/awardee-type", "valueString": "DV" }], "identifier": [{ "system": "http://all-of-us.org/fhir/sites/awardee-id", "value": "TEST_HPO_NAME" }], "active": True, "type": [{ "coding": [{ "code": "AWARDEE", "system": "http://all-of-us.org/fhir/sites/type" }] }], "name": "Test new HPO display name" } self.send_put('organization/hierarchy', request_data=request_json) result = self.send_get('Awardee/TEST_HPO_NAME') self.assertEquals( _make_awardee_resource('TEST_HPO_NAME', 'Test new HPO display name', 'DV'), result) def test_update_existing_hpo(self): self._setup_data() request_json = { "resourceType": "Organization", "id": "a893282c-2717-4a20-b276-d5c9c2c0e51f", 'meta': { 'versionId': '2' }, "extension": [{ "url": "http://all-of-us.org/fhir/sites/awardee-type", "valueString": "DV" }], "identifier": [{ "system": "http://all-of-us.org/fhir/sites/awardee-id", "value": "PITT" }], "active": True, "type": [{ "coding": [{ "code": "AWARDEE", "system": "http://all-of-us.org/fhir/sites/type" }] }], "name": "Test update HPO display name" } self.send_put('organization/hierarchy', request_data=request_json) result = self.send_get('Awardee/PITT') self.assertEqual(result['displayName'], 'Test update HPO display name') self.assertEqual(result['type'], 'DV') def test_create_new_organization(self): self._setup_data() request_json = { "resourceType": "Organization", "id": "a893282c-2717-4a20-b276-d5c9c2c0e51f", 'meta': { 'versionId': '1' }, "extension": [], "identifier": [{ "system": "http://all-of-us.org/fhir/sites/organization-id", "value": "TEST_NEW_ORG" }], "active": True, "type": [{ "coding": [{ "code": "ORGANIZATION", "system": "http://all-of-us.org/fhir/sites/type" }] }], "name": "Test create organization display name", "partOf": { "reference": "Organization/h123457" } } result_before = self.send_get('Awardee/PITT') self.assertEqual(2, len(result_before['organizations'])) self.send_put('organization/hierarchy', request_data=request_json) result_after = self.send_get('Awardee/PITT') self.assertEqual(3, len(result_after['organizations'])) self.assertIn( { 'displayName': 'Test create organization display name', 'id': 'TEST_NEW_ORG' }, result_after['organizations']) def test_update_existing_organization(self): self._setup_data() request_json = { "resourceType": "Organization", "id": "a893282c-2717-4a20-b276-d5c9c2c0e51f", 'meta': { 'versionId': '2' }, "extension": [], "identifier": [{ "system": "http://all-of-us.org/fhir/sites/organization-id", "value": "AARDVARK_ORG" }], "active": True, "type": [{ "coding": [{ "code": "ORGANIZATION", "system": "http://all-of-us.org/fhir/sites/type" }] }], "name": "Test update organization display name", "partOf": { "reference": "Organization/h123457" } } result_before = self.send_get('Awardee/PITT') self.assertEqual(2, len(result_before['organizations'])) self.send_put('organization/hierarchy', request_data=request_json) result_after = self.send_get('Awardee/PITT') self.assertEqual(2, len(result_after['organizations'])) self.assertIn( { 'displayName': 'Test update organization display name', 'id': 'AARDVARK_ORG' }, result_after['organizations']) @mock.patch( 'dao.organization_hierarchy_sync_dao.OrganizationHierarchySyncDao.' '_get_lat_long_for_site') @mock.patch( 'dao.organization_hierarchy_sync_dao.OrganizationHierarchySyncDao.' '_get_time_zone') def test_create_new_site(self, time_zone, lat_long): self._setup_data() lat_long.return_value = 100, 110 time_zone.return_value = 'America/Los_Angeles' request_json = { "resourceType": "Organization", "id": "a893282c-2717-4a20-b276-d5c9c2c0e51f", 'meta': { 'versionId': '1' }, "extension": [{ "url": "http://all-of-us.org/fhir/sites/enrollmentStatusActive", "valueBoolean": True }, { "url": "http://all-of-us.org/fhir/sites/digitalSchedulingStatusActive", "valueBoolean": True }, { "url": "http://all-of-us.org/fhir/sites/schedulingStatusActive", "valueBoolean": True }, { "url": "http://all-of-us.org/fhir/sites/notes", "valueString": "This is a note about an organization" }, { "url": "http://all-of-us.org/fhir/sites/schedulingInstructions", "valueString": "Please schedule appointments up to a week before intended date." }, { "url": "http://all-of-us.org/fhir/sites/anticipatedLaunchDate", "valueDate": "07-02-2010" }, { "url": "http://all-of-us.org/fhir/sites/locationName", "valueString": "Thompson Building" }, { "url": "http://all-of-us.org/fhir/sites/directions", "valueString": "Exit 95 N and make a left onto Fake Street" }], "identifier": [{ "system": "http://all-of-us.org/fhir/sites/site-id", "value": "hpo-site-awesome-testing" }, { "system": "http://all-of-us.org/fhir/sites/mayo-link-identifier", "value": "123456" }, { "system": "http://all-of-us.org/fhir/sites/google-group-identifier", "value": "Awesome Genomics Testing" }], "active": True, "type": [{ "coding": [{ "code": "SITE", "system": "http://all-of-us.org/fhir/sites/type" }] }], "name": "Awesome Genomics Testing", "partOf": { "reference": "Organization/o123457" }, "address": [{ "line": ["1855 4th Street", "AAC5/6"], "city": "San Francisco", "state": "CA", "postalCode": "94158" }], "contact": [{ "telecom": [{ "system": "phone", "value": "7031234567" }] }, { "telecom": [{ "system": "email", "value": "*****@*****.**" }] }, { "telecom": [{ "system": "url", "value": "http://awesome-genomic-testing.com" }] }] } self.send_put('organization/hierarchy', request_data=request_json) self.send_get('Awardee/PITT') existing_map = { entity.googleGroup: entity for entity in self.site_dao.get_all() } existing_entity = existing_map.get('hpo-site-awesome-testing') self.assertEqual(existing_entity.adminEmails, '*****@*****.**') self.assertEqual(existing_entity.siteStatus, SiteStatus('ACTIVE')) self.assertEqual(existing_entity.isObsolete, None) self.assertEqual(existing_entity.city, 'San Francisco') self.assertEqual(existing_entity.googleGroup, 'hpo-site-awesome-testing') self.assertEqual(existing_entity.state, 'CA') self.assertEqual(existing_entity.digitalSchedulingStatus, DigitalSchedulingStatus('ACTIVE')) self.assertEqual(existing_entity.mayolinkClientNumber, 123456) self.assertEqual(existing_entity.address1, '1855 4th Street') self.assertEqual(existing_entity.address2, 'AAC5/6') self.assertEqual(existing_entity.zipCode, '94158') self.assertEqual(existing_entity.directions, 'Exit 95 N and make a left onto Fake Street') self.assertEqual(existing_entity.notes, 'This is a note about an organization') self.assertEqual(existing_entity.enrollingStatus, EnrollingStatus('ACTIVE')) self.assertEqual( existing_entity.scheduleInstructions, 'Please schedule appointments up to a week before intended date.') self.assertEqual(existing_entity.physicalLocationName, 'Thompson Building') self.assertEqual(existing_entity.link, 'http://awesome-genomic-testing.com') self.assertEqual(existing_entity.launchDate, datetime.date(2010, 7, 2)) self.assertEqual(existing_entity.phoneNumber, '7031234567') @mock.patch( 'dao.organization_hierarchy_sync_dao.OrganizationHierarchySyncDao.' '_get_lat_long_for_site') @mock.patch( 'dao.organization_hierarchy_sync_dao.OrganizationHierarchySyncDao.' '_get_time_zone') def test_update_existing_site(self, time_zone, lat_long): self._setup_data() lat_long.return_value = 100, 110 time_zone.return_value = 'America/Los_Angeles' request_json = { "resourceType": "Organization", "id": "a893282c-2717-4a20-b276-d5c9c2c0e51f", 'meta': { 'versionId': '2' }, "extension": [{ "url": "http://all-of-us.org/fhir/sites/enrollmentStatusActive", "valueBoolean": True }, { "url": "http://all-of-us.org/fhir/sites/digitalSchedulingStatusActive", "valueBoolean": False }, { "url": "http://all-of-us.org/fhir/sites/schedulingStatusActive", "valueBoolean": True }, { "url": "http://all-of-us.org/fhir/sites/notes", "valueString": "This is a note about an organization" }, { "url": "http://all-of-us.org/fhir/sites/schedulingInstructions", "valueString": "Please schedule appointments up to a week before intended date." }, { "url": "http://all-of-us.org/fhir/sites/anticipatedLaunchDate", "valueDate": "07-02-2010" }, { "url": "http://all-of-us.org/fhir/sites/locationName", "valueString": "Thompson Building" }, { "url": "http://all-of-us.org/fhir/sites/directions", "valueString": "Exit 95 N and make a left onto Fake Street" }], "identifier": [{ "system": "http://all-of-us.org/fhir/sites/site-id", "value": "hpo-site-1" }, { "system": "http://all-of-us.org/fhir/sites/mayo-link-identifier", "value": "123456" }, { "system": "http://all-of-us.org/fhir/sites/google-group-identifier", "value": "Awesome Genomics Testing" }], "active": True, "type": [{ "coding": [{ "code": "SITE", "system": "http://all-of-us.org/fhir/sites/type" }] }], "name": "Awesome Genomics Testing", "partOf": { "reference": "Organization/o123456" }, "address": [{ "line": ["1855 4th Street", "AAC5/6"], "city": "San Francisco", "state": "CA", "postalCode": "94158" }], "contact": [{ "telecom": [{ "system": "phone", "value": "7031234567" }] }, { "telecom": [{ "system": "email", "value": "*****@*****.**" }] }, { "telecom": [{ "system": "url", "value": "http://awesome-genomic-testing.com" }] }] } self.send_put('organization/hierarchy', request_data=request_json) existing_map = { entity.googleGroup: entity for entity in self.site_dao.get_all() } existing_entity = existing_map.get('hpo-site-1') self.assertEqual(existing_entity.adminEmails, '*****@*****.**') self.assertEqual(existing_entity.siteStatus, SiteStatus('ACTIVE')) self.assertEqual(existing_entity.isObsolete, None) self.assertEqual(existing_entity.city, 'San Francisco') self.assertEqual(existing_entity.googleGroup, 'hpo-site-1') self.assertEqual(existing_entity.state, 'CA') self.assertEqual(existing_entity.digitalSchedulingStatus, DigitalSchedulingStatus('INACTIVE')) self.assertEqual(existing_entity.mayolinkClientNumber, 123456) self.assertEqual(existing_entity.address1, '1855 4th Street') self.assertEqual(existing_entity.address2, 'AAC5/6') self.assertEqual(existing_entity.zipCode, '94158') self.assertEqual(existing_entity.directions, 'Exit 95 N and make a left onto Fake Street') self.assertEqual(existing_entity.notes, 'This is a note about an organization') self.assertEqual(existing_entity.enrollingStatus, EnrollingStatus('ACTIVE')) self.assertEqual( existing_entity.scheduleInstructions, 'Please schedule appointments up to a week before intended date.') self.assertEqual(existing_entity.physicalLocationName, 'Thompson Building') self.assertEqual(existing_entity.link, 'http://awesome-genomic-testing.com') self.assertEqual(existing_entity.launchDate, datetime.date(2010, 7, 2)) self.assertEqual(existing_entity.phoneNumber, '7031234567') @mock.patch( 'dao.organization_hierarchy_sync_dao.OrganizationHierarchySyncDao.' '_get_lat_long_for_site') @mock.patch( 'dao.organization_hierarchy_sync_dao.OrganizationHierarchySyncDao.' '_get_time_zone') def test_create_hpo_org_site(self, time_zone, lat_long): hpo_json = { "resourceType": "Organization", "id": "a893282c-2717-4a20-b276-d5c9c2c0e51f", 'meta': { 'versionId': '1' }, "extension": [{ "url": "http://all-of-us.org/fhir/sites/awardee-type", "valueString": "DV" }], "identifier": [{ "system": "http://all-of-us.org/fhir/sites/awardee-id", "value": "TEST_HPO_NAME" }], "active": True, "type": [{ "coding": [{ "code": "AWARDEE", "system": "http://all-of-us.org/fhir/sites/type" }] }], "name": "Test new HPO display name" } self.send_put('organization/hierarchy', request_data=hpo_json) org_json = { "resourceType": "Organization", "id": "a893282c-2717-4a20-b276-d5c9c2c0e123", 'meta': { 'versionId': '1' }, "extension": [], "identifier": [{ "system": "http://all-of-us.org/fhir/sites/organization-id", "value": "TEST_NEW_ORG" }], "active": True, "type": [{ "coding": [{ "code": "ORGANIZATION", "system": "http://all-of-us.org/fhir/sites/type" }] }], "name": "Test create organization display name", "partOf": { "reference": "Organization/a893282c-2717-4a20-b276-d5c9c2c0e51f" } } self.send_put('organization/hierarchy', request_data=org_json) lat_long.return_value = 100, 110 time_zone.return_value = 'America/Los_Angeles' site_json = { "resourceType": "Organization", "id": "a893282c-2717-4a20-b276-d5c9c2c0e234", 'meta': { 'versionId': '1' }, "extension": [{ "url": "http://all-of-us.org/fhir/sites/enrollmentStatusActive", "valueBoolean": True }, { "url": "http://all-of-us.org/fhir/sites/digitalSchedulingStatusActive", "valueBoolean": True }, { "url": "http://all-of-us.org/fhir/sites/schedulingStatusActive", "valueBoolean": True }, { "url": "http://all-of-us.org/fhir/sites/notes", "valueString": "This is a note about an organization" }, { "url": "http://all-of-us.org/fhir/sites/schedulingInstructions", "valueString": "Please schedule appointments up to a week before intended date." }, { "url": "http://all-of-us.org/fhir/sites/anticipatedLaunchDate", "valueDate": "07-02-2010" }, { "url": "http://all-of-us.org/fhir/sites/locationName", "valueString": "Thompson Building" }, { "url": "http://all-of-us.org/fhir/sites/directions", "valueString": "Exit 95 N and make a left onto Fake Street" }], "identifier": [{ "system": "http://all-of-us.org/fhir/sites/site-id", "value": "hpo-site-awesome-testing" }, { "system": "http://all-of-us.org/fhir/sites/mayo-link-identifier", "value": "123456" }, { "system": "http://all-of-us.org/fhir/sites/google-group-identifier", "value": "Awesome Genomics Testing" }], "active": True, "type": [{ "coding": [{ "code": "SITE", "system": "http://all-of-us.org/fhir/sites/type" }] }], "name": "Awesome Genomics Testing", "partOf": { "reference": "Organization/a893282c-2717-4a20-b276-d5c9c2c0e123" }, "address": [{ "line": ["1855 4th Street", "AAC5/6"], "city": "San Francisco", "state": "CA", "postalCode": "94158" }], "contact": [{ "telecom": [{ "system": "phone", "value": "7031234567" }] }, { "telecom": [{ "system": "email", "value": "*****@*****.**" }] }, { "telecom": [{ "system": "url", "value": "http://awesome-genomic-testing.com" }] }] } self.send_put('organization/hierarchy', request_data=site_json) result = self.send_get('Awardee/TEST_HPO_NAME') self.assertEqual( { u'displayName': u'Test new HPO display name', u'type': u'DV', u'id': u'TEST_HPO_NAME', u'organizations': [{ u'displayName': u'Test create organization display name', u'id': u'TEST_NEW_ORG', u'sites': [{ u'mayolinkClientNumber': 123456, u'timeZoneId': u'America/Los_Angeles', u'displayName': u'Awesome Genomics Testing', u'notes': u'This is a note about an organization', u'launchDate': u'2010-07-02', u'notesEs': u'', u'enrollingStatus': u'ACTIVE', u'longitude': 110.0, u'schedulingInstructions': u'Please schedule appointments up ' u'to a week before intended date.', u'latitude': 100.0, u'physicalLocationName': u'Thompson Building', u'phoneNumber': u'7031234567', u'siteStatus': u'ACTIVE', u'address': { u'postalCode': u'94158', u'city': u'San Francisco', u'line': [u'1855 4th Street', u'AAC5/6'], u'state': u'CA' }, u'directions': u'Exit 95 N and make a left onto Fake Street', u'link': u'http://awesome-genomic-testing.com', u'id': u'hpo-site-awesome-testing', u'adminEmails': [u'*****@*****.**'], u'digitalSchedulingStatus': u'ACTIVE' }] }] }, result) def _setup_data(self): organization_dao = OrganizationDao() site_dao = SiteDao() org_1 = organization_dao.insert( Organization(externalId='ORG_1', displayName='Organization 1', hpoId=PITT_HPO_ID, resourceId='o123456')) organization_dao.insert( Organization(externalId='AARDVARK_ORG', displayName='Aardvarks Rock', hpoId=PITT_HPO_ID, resourceId='o123457')) site_dao.insert( Site(siteName='Site 1', googleGroup='hpo-site-1', mayolinkClientNumber=123456, organizationId=org_1.organizationId, siteStatus=SiteStatus.ACTIVE, enrollingStatus=EnrollingStatus.ACTIVE, launchDate=datetime.datetime(2016, 1, 1), notes='notes', latitude=12.1, longitude=13.1, directions='directions', physicalLocationName='locationName', address1='address1', address2='address2', city='Austin', state='TX', zipCode='78751', phoneNumber='555-555-5555', adminEmails='[email protected], [email protected]', link='http://www.example.com')) site_dao.insert( Site(siteName='Zebras Rock', googleGroup='aaaaaaa', organizationId=org_1.organizationId, enrollingStatus=EnrollingStatus.INACTIVE, siteStatus=SiteStatus.INACTIVE))