def main(args):
skip_count = 0
new_or_updated_count = 0
matched_count = 0
with open(args.file, 'r') as csv_file:
sites_reader = csv.DictReader(csv_file)
hpo_dao = HPODao()
site_dao = SiteDao()
existing_site_map = {
site.googleGroup: site
for site in site_dao.get_all()
}
with site_dao.session() as session:
for row in sites_reader:
site = _site_from_row(row, hpo_dao)
if site is None:
skip_count += 1
continue
changed = _upsert_site(site,
existing_site_map.get(site.googleGroup),
site_dao, session, args.dry_run)
if changed:
new_or_updated_count += 1
else:
matched_count += 1
logging.info(
'Done%s. %d skipped, %d sites new/updated, %d sites not changed.',
' (dry run)' if args.dry_run else '', skip_count, new_or_updated_count,
matched_count)
def __init__(self):
args = parser.parse_args()
self.organization_dao = OrganizationDao()
self.stub_geocoding = args.stub_geocoding
self.ACTIVE = SiteStatus.ACTIVE
self.status_exception_list = ['hpo-site-walgreensphoenix']
self.instance = args.instance
self.creds_file = args.creds_file
self.new_sites_list = []
self.project = None
if args.project:
self.project = args.project
if self.project in ENV_LIST:
self.environment = ' ' + self.project.split('-')[-1].upper()
else:
self.environment = ' ' + ENV_TEST.split('-')[-1].upper()
super(SiteImporter,
self).__init__('site', SiteDao(), 'siteId', 'googleGroup', [
SITE_ORGANIZATION_ID_COLUMN, SITE_SITE_ID_COLUMN,
SITE_SITE_COLUMN, SITE_STATUS_COLUMN + self.environment,
ENROLLING_STATUS_COLUMN + self.environment,
DIGITAL_SCHEDULING_STATUS_COLUMN + self.environment
])
def add_missing_hpo_from_site(self, session, participant_id, site_id):
if site_id is None:
raise BadRequest('No site ID given for auto-pairing participant.')
site = SiteDao().get_with_session(session, site_id)
if site is None:
raise BadRequest('Invalid siteId reference %r.' % site_id)
participant = self.get_for_update(session, participant_id)
if participant is None:
raise BadRequest('No participant %r for HPO ID udpate.' %
participant_id)
if participant.siteId == site.siteId:
return
participant.hpoId = site.hpoId
participant.organizationId = site.organizationId
participant.siteId = site.siteId
participant.providerLink = make_primary_provider_link_for_id(
site.hpoId)
if participant.participantSummary is None:
raise RuntimeError('No ParticipantSummary available for P%d.' %
participant_id)
participant.participantSummary.hpoId = site.hpoId
participant.lastModified = clock.CLOCK.now()
# Update the version and add history row
self._do_update(session, participant, participant)
def test_overwrite_existing_pairing(self):
participant_id = 99
created = self.dao.insert(
Participant(participantId=participant_id,
biobankId=2,
hpoId=self._test_db.hpo_id,
providerLink=make_primary_provider_link_for_id(
self._test_db.hpo_id)))
self.participant_summary_dao.insert(self.participant_summary(created))
self.assertEquals(created.hpoId, self._test_db.hpo_id) # sanity check
other_hpo = HPODao().insert(
HPO(hpoId=PITT_HPO_ID + 1, name='DIFFERENT_HPO'))
other_site = SiteDao().insert(
Site(hpoId=other_hpo.hpoId,
siteName='Arbitrary Site',
googleGroup='*****@*****.**'))
with self.dao.session() as session:
self.dao.add_missing_hpo_from_site(session, participant_id,
other_site.siteId)
# Original Participant + summary is affected.
refetched = self.dao.get(participant_id)
self.assertEquals(refetched.hpoId, other_hpo.hpoId)
self.assertEquals(refetched.providerLink,
make_primary_provider_link_for_id(other_hpo.hpoId))
self.assertEquals(
self.participant_summary_dao.get(participant_id).hpoId,
other_hpo.hpoId)
def setUp(self, **kwargs):
super(MetricsEhrApiTestBase, self).setUp(use_mysql=True, **kwargs)
self.dao = ParticipantDao()
self.ps_dao = ParticipantSummaryDao()
self.ehr_receipt_dao = EhrReceiptDao()
self.ps = ParticipantSummary()
self.calendar_dao = CalendarDao()
self.site_dao = SiteDao()
self.hpo_dao = HPODao()
self.org_dao = OrganizationDao()
self.hpo_test = self._make_hpo(hpoId=TEST_HPO_ID,
name=TEST_HPO_NAME,
displayName='Test',
organizationType=OrganizationType.UNSET)
self.hpo_foo = self._make_hpo(hpoId=10, name='FOO', displayName='Foo')
self.hpo_bar = self._make_hpo(hpoId=11, name='BAR', displayName='Bar')
self.org_foo_a = self._make_org(organizationId=10,
externalId='FOO_A',
displayName='Foo A',
hpoId=self.hpo_foo.hpoId)
self.org_bar_a = self._make_org(organizationId=11,
externalId='BAR_A',
displayName='Bar A',
hpoId=self.hpo_bar.hpoId)
def __init__(self):
super(ParticipantSummaryDao,
self).__init__(ParticipantSummary,
order_by_ending=_ORDER_BY_ENDING)
self.hpo_dao = HPODao()
self.code_dao = CodeDao()
self.site_dao = SiteDao()
self.organization_dao = OrganizationDao()
def get_site_id_by_site_value(obj):
if 'site' in obj:
from dao.site_dao import SiteDao
site_dao = SiteDao()
site = site_dao.get_by_google_group(obj['site']['value'])
if site is not None:
return site.siteId
return None
def setUp(self, **kwargs):
super(SyncConsentFilesTest, self).setUp(use_mysql=True, **kwargs)
NdbTestBase.doSetUp(self)
TestBase.setup_fake(self)
self.org_dao = OrganizationDao()
self.site_dao = SiteDao()
self.participant_dao = ParticipantDao()
self.summary_dao = ParticipantSummaryDao()
def get_location_site_id(location_value):
if not location_value.startswith(_LOCATION_PREFIX):
logging.warn("Invalid location: %s" % location_value)
return None
google_group = location_value[len(_LOCATION_PREFIX):]
site = SiteDao().get_by_google_group(google_group)
if not site:
logging.warn("Unknown site: %s" % google_group)
return None
return site.siteId
def _setup_hpos(self, org_dao=None):
hpo_dao = HPODao()
hpo_dao.insert(
HPO(hpoId=UNSET_HPO_ID,
name='UNSET',
displayName='Unset',
organizationType=OrganizationType.UNSET))
hpo_dao.insert(
HPO(hpoId=PITT_HPO_ID,
name='PITT',
displayName='Pittsburgh',
organizationType=OrganizationType.HPO))
hpo_dao.insert(
HPO(hpoId=AZ_HPO_ID,
name='AZ_TUCSON',
displayName='Arizona',
organizationType=OrganizationType.HPO))
self.hpo_id = PITT_HPO_ID
org_dao = OrganizationDao()
org_dao.insert(
Organization(organizationId=AZ_ORG_ID,
externalId='AZ_TUCSON_BANNER_HEALTH',
displayName='Banner Health',
hpoId=AZ_HPO_ID))
created_org = org_dao.insert(
Organization(organizationId=PITT_ORG_ID,
externalId='PITT_BANNER_HEALTH',
displayName='PITT display Banner Health',
hpoId=PITT_HPO_ID))
self.organization_id = created_org.organizationId
site_dao = SiteDao()
created_site = site_dao.insert(
Site(siteName='Monroeville Urgent Care Center',
googleGroup='hpo-site-monroeville',
mayolinkClientNumber=7035769,
organizationId=PITT_ORG_ID,
hpoId=PITT_HPO_ID))
self.site_id = created_site.siteId
site_dao.insert(
Site(siteName='Phoenix Urgent Care Center',
googleGroup='hpo-site-bannerphoenix',
mayolinkClientNumber=7035770,
organizationId=PITT_ORG_ID,
hpoId=PITT_HPO_ID))
site_dao.insert(
Site(siteName='Phoenix clinic',
googleGroup='hpo-site-clinic-phoenix',
mayolinkClientNumber=7035770,
organizationId=AZ_ORG_ID,
hpoId=AZ_HPO_ID))
def __init__(self):
super(SiteImporter,
self).__init__('site', SiteDao(), 'siteId', 'googleGroup', [
SITE_ORGANIZATION_ID_COLUMN, SITE_SITE_ID_COLUMN,
SITE_SITE_COLUMN, SITE_STATUS_COLUMN, ENROLLING_STATUS_COLUMN
])
self.organization_dao = OrganizationDao()
args = parser.parse_args()
self.geocode_flag = args.geocode_flag
self.ACTIVE = SiteStatus.ACTIVE
self.status_exception_list = ['hpo-site-walgreensphoenix']
def __init__(self, client, use_local_files=None):
self._use_local_files = use_local_files
self._client = client
self._hpos = HPODao().get_all()
self._sites = SiteDao().get_all()
if not self._sites:
raise BadRequest('No sites found; import sites before running generator.')
self._now = clock.CLOCK.now()
self._consent_questionnaire_id_and_version = None
self._setup_data()
self._setup_questionnaires()
self._min_birth_date = self._now - datetime.timedelta(days=_MAX_PARTICIPANT_AGE * 365)
self._max_days_for_birth_date = 365 * (_MAX_PARTICIPANT_AGE - _MIN_PARTICIPANT_AGE)
def _setup_unset_enrollment_site(self):
site_dao = SiteDao()
organization_dao = OrganizationDao()
org_2 = organization_dao.insert(
Organization(externalId='ORG_2',
displayName='Organization 2',
hpoId=PITT_HPO_ID))
site_dao.insert(
Site(siteName='not enrolling site',
googleGroup='not_enrolling_dot_com',
organizationId=org_2.organizationId,
enrollingStatus=EnrollingStatus.UNSET,
siteStatus=SiteStatus.INACTIVE))
def _to_handling_info(self, username, site_id):
if not username and not site_id:
return None
info = _FhirBiobankOrderHandlingInfo()
if site_id:
site = SiteDao().get(site_id)
info.site = Identifier()
info.site.system = SITE_ID_SYSTEM
info.site.value = site.googleGroup
if username:
info.author = Identifier()
info.author.system = HEALTHPRO_USERNAME_SYSTEM
info.author.value = username
return info
def _parse_handling_info(self, handling_info):
site_id = None
username = None
if handling_info.site:
if handling_info.site.system != SITE_ID_SYSTEM:
raise BadRequest('Invalid site system: %s' %
handling_info.site.system)
site = SiteDao().get_by_google_group(handling_info.site.value)
if not site:
raise BadRequest('Unrecognized site: %s' %
handling_info.site.value)
site_id = site.siteId
if handling_info.author:
if handling_info.author.system != HEALTHPRO_USERNAME_SYSTEM:
raise BadRequest('Invalid author system: %s' %
handling_info.author.system)
username = handling_info.author.value
return username, site_id
def _setup_data(self):
organization_dao = OrganizationDao()
site_dao = SiteDao()
org_1 = organization_dao.insert(
Organization(externalId='ORG_1',
displayName='Organization 1',
hpoId=PITT_HPO_ID,
resourceId='o123456'))
organization_dao.insert(
Organization(externalId='AARDVARK_ORG',
displayName='Aardvarks Rock',
hpoId=PITT_HPO_ID,
resourceId='o123457'))
site_dao.insert(
Site(siteName='Site 1',
googleGroup='hpo-site-1',
mayolinkClientNumber=123456,
organizationId=org_1.organizationId,
siteStatus=SiteStatus.ACTIVE,
enrollingStatus=EnrollingStatus.ACTIVE,
launchDate=datetime.datetime(2016, 1, 1),
notes='notes',
latitude=12.1,
longitude=13.1,
directions='directions',
physicalLocationName='locationName',
address1='address1',
address2='address2',
city='Austin',
state='TX',
zipCode='78751',
phoneNumber='555-555-5555',
adminEmails='[email protected], [email protected]',
link='http://www.example.com'))
site_dao.insert(
Site(siteName='Zebras Rock',
googleGroup='aaaaaaa',
organizationId=org_1.organizationId,
enrollingStatus=EnrollingStatus.INACTIVE,
siteStatus=SiteStatus.INACTIVE))
def _cleanup_old_entities(self, session, row_list, dry_run):
log_prefix = '(dry run) ' if dry_run else ''
self.site_dao = SiteDao()
existing_sites = set(site.googleGroup
for site in self.site_dao.get_all())
site_group_list_from_sheet = [
str(row[SITE_SITE_ID_COLUMN].lower()) for row in row_list
]
sites_to_remove = existing_sites - set(site_group_list_from_sheet)
if not sites_to_remove:
return
site_id_list = []
for site in sites_to_remove:
old_site = self.site_dao.get_by_google_group(site)
if old_site and old_site.isObsolete != ObsoleteStatus.OBSOLETE:
logging.info(
log_prefix +
'Deleting old Site no longer in Google sheet: %s', site)
site_id_list.append(old_site.siteId)
self.deletion_count += 1
elif old_site and old_site.isObsolete == ObsoleteStatus.OBSOLETE:
logging.info(
'Not attempting to delete site [%s] with existing obsolete status',
old_site.googleGroup)
if dry_run or not site_id_list:
return
# Set obsolete first, in case the sites cannot be deleted below.
str_list = ','.join([str(i) for i in site_id_list])
sql = """ UPDATE site
SET is_obsolete = 1
WHERE site_id in ({site_id_list})""".format(site_id_list=str_list)
session.execute(sql)
self.site_dao._invalidate_cache()
# Try to delete old sites, this may partially fail if there are still foreign references.
self.delete_sql_statement(session, str_list)
def _setup_hpos(self):
hpo_dao = HPODao()
hpo_dao.insert(HPO(hpoId=UNSET_HPO_ID, name='UNSET'))
hpo_dao.insert(
HPO(hpoId=PITT_HPO_ID,
name='PITT',
organizationType=OrganizationType.HPO))
self.hpo_id = PITT_HPO_ID
site_dao = SiteDao()
created_site = site_dao.insert(
Site(siteName='Monroeville Urgent Care Center',
googleGroup='hpo-site-monroeville',
consortiumName='Pittsburgh',
mayolinkClientNumber=7035769,
hpoId=PITT_HPO_ID))
self.site_id = created_site.siteId
site_dao.insert(
Site(siteName='Phoenix Urgent Care Center',
googleGroup='hpo-site-bannerphoenix',
consortiumName='Pittsburgh',
mayolinkClientNumber=7035770,
hpoId=PITT_HPO_ID))
def setUp(self):
super(HierarchyContentApiTest, self).setUp(with_data=False)
hpo_dao = HPODao()
hpo_dao.insert(
HPO(hpoId=UNSET_HPO_ID,
name='UNSET',
displayName='Unset',
organizationType=OrganizationType.UNSET,
resourceId='h123456'))
hpo_dao.insert(
HPO(hpoId=PITT_HPO_ID,
name='PITT',
displayName='Pittsburgh',
organizationType=OrganizationType.HPO,
resourceId='h123457'))
hpo_dao.insert(
HPO(hpoId=AZ_HPO_ID,
name='AZ_TUCSON',
displayName='Arizona',
organizationType=OrganizationType.HPO,
resourceId='h123458'))
self.site_dao = SiteDao()
self.org_dao = OrganizationDao()
def __init__(self):
super(ParticipantDao, self).__init__(Participant)
self.hpo_dao = HPODao()
self.organization_dao = OrganizationDao()
self.site_dao = SiteDao()
def setUp(self):
super(SiteDaoTest, self).setUp()
self.site_dao = SiteDao()
self.participant_dao = ParticipantDao()
self.ps_dao = ParticipantSummaryDao()
self.ps_history = ParticipantHistoryDao()
def __init__(self):
super(OrganizationHierarchySyncDao, self).__init__(HPO)
self.hpo_dao = HPODao()
self.organization_dao = OrganizationDao()
self.site_dao = SiteDao()
def setUp(self):
super(SiteDaoTest, self).setUp()
self.site_dao = SiteDao()