def test_insert(self): organization = Organization(externalId='myorg', displayName='myorg_display', hpoId=PITT_HPO_ID, isObsolete=1) created_organization = self.organization_dao.insert(organization) new_organization = self.organization_dao.get(created_organization.organizationId) organization.organizationId = created_organization.organizationId organization.isObsolete = new_organization.isObsolete self.assertEquals(organization.asdict(), new_organization.asdict())
def _setup_hpos(self, org_dao=None): hpo_dao = HPODao() hpo_dao.insert( HPO(hpoId=UNSET_HPO_ID, name='UNSET', displayName='Unset', organizationType=OrganizationType.UNSET)) hpo_dao.insert( HPO(hpoId=PITT_HPO_ID, name='PITT', displayName='Pittsburgh', organizationType=OrganizationType.HPO)) hpo_dao.insert( HPO(hpoId=AZ_HPO_ID, name='AZ_TUCSON', displayName='Arizona', organizationType=OrganizationType.HPO)) self.hpo_id = PITT_HPO_ID org_dao = OrganizationDao() org_dao.insert( Organization(organizationId=AZ_ORG_ID, externalId='AZ_TUCSON_BANNER_HEALTH', displayName='Banner Health', hpoId=AZ_HPO_ID)) created_org = org_dao.insert( Organization(organizationId=PITT_ORG_ID, externalId='PITT_BANNER_HEALTH', displayName='PITT display Banner Health', hpoId=PITT_HPO_ID)) self.organization_id = created_org.organizationId site_dao = SiteDao() created_site = site_dao.insert( Site(siteName='Monroeville Urgent Care Center', googleGroup='hpo-site-monroeville', mayolinkClientNumber=7035769, organizationId=PITT_ORG_ID, hpoId=PITT_HPO_ID)) self.site_id = created_site.siteId site_dao.insert( Site(siteName='Phoenix Urgent Care Center', googleGroup='hpo-site-bannerphoenix', mayolinkClientNumber=7035770, organizationId=PITT_ORG_ID, hpoId=PITT_HPO_ID)) site_dao.insert( Site(siteName='Phoenix clinic', googleGroup='hpo-site-clinic-phoenix', mayolinkClientNumber=7035770, organizationId=AZ_ORG_ID, hpoId=AZ_HPO_ID))
def _update_organization(self, hierarchy_org_obj): if hierarchy_org_obj.id is None: raise BadRequest('No id found in payload data.') organization_id = self._get_value_from_identifier( hierarchy_org_obj, HIERARCHY_CONTENT_SYSTEM_PREFIX + 'organization-id') if organization_id is None: raise BadRequest( 'No organization-identifier info found in payload data.') is_obsolete = ObsoleteStatus( 'OBSOLETE') if not hierarchy_org_obj.active else None resource_id = self._get_reference(hierarchy_org_obj) hpo = self.hpo_dao.get_by_resource_id(resource_id) if hpo is None: raise BadRequest( 'Invalid partOf reference {} importing organization {}'.format( resource_id, organization_id)) entity = Organization(externalId=organization_id.upper(), displayName=hierarchy_org_obj.name, hpoId=hpo.hpoId, isObsolete=is_obsolete, resourceId=hierarchy_org_obj.id) existing_map = { entity.externalId: entity for entity in self.organization_dao.get_all() } existing_entity = existing_map.get(entity.externalId) with self.organization_dao.session() as session: if existing_entity: new_dict = entity.asdict() new_dict['organizationId'] = None existing_dict = existing_entity.asdict() existing_dict['organizationId'] = None if existing_dict == new_dict: logging.info('Not updating {}.'.format( new_dict['externalId'])) else: existing_entity.displayName = entity.displayName existing_entity.hpoId = entity.hpoId existing_entity.isObsolete = entity.isObsolete existing_entity.resourceId = entity.resourceId self.organization_dao.update_with_session( session, existing_entity) else: self.organization_dao.insert_with_session(session, entity) org_id = self.organization_dao.get_by_external_id( organization_id.upper()).organizationId bq_organization_update_by_id(org_id)
def _create_org(self, id_): org = Organization(organizationId=id_, externalId=id_, displayName=id_, hpoId=UNSET_HPO_ID) self.org_dao.insert(org) return org
def _make_org(self, hpo, int_id, external_id): org = Organization(organizationId=int_id, externalId=external_id, displayName='SOME ORG', hpoId=hpo.hpoId) self.org_dao.insert(org) return org
def init(): handler = pugsql.module('assets/sql') handler.connect('sqlite:///memory') settings.sql = handler # Create table if they don't exist handler.org_create() handler.user_create() handler.prof_create() handler.fund_create() handler.stat_create() handler.value_create() handler.role_create() handler.role_user_create() # Add test rows if they don't exist org = handler.org_find(id=1) if (org == None): org = Organization.add('Test Company') admin = User.add('admin', 'test123') guest = User.add('guest', 'test123') admin_role = Role.add('admin') admin.add_to_role(role_id=admin_role.id) profile = Profile.add('Test Profile', org.id) fund = Fund.add('Test Fund', 'Test Manager', 2000, 0.00, 0.00, profile.id) # return the queries handler return handler
def test_participant_pairing_updates_onchange(self): provider_link = '[{"organization": {"reference": "Organization/AZ_TUCSON"}, "primary": true}]' TIME = datetime.datetime(2018, 1, 1) TIME2 = datetime.datetime(2018, 1, 2) insert_org = self.organization_dao.insert( Organization(externalId='tardis', displayName='bluebox', hpoId=PITT_HPO_ID)) with FakeClock(TIME): self.participant_dao.insert(Participant(participantId=1, biobankId=2)) participant = self.participant_dao.get(1) participant.organizationId = insert_org.organizationId self.participant_dao.update(participant) self.assertEquals(participant.hpoId, insert_org.hpoId) participant = self.participant_dao.get(1) p_summary = self.ps_dao.insert(self.participant_summary(participant)) with FakeClock(TIME2): insert_org.hpoId = AZ_HPO_ID self.organization_dao.update(insert_org) new_org = self.organization_dao.get_by_external_id('tardis') ps = self.ps_dao.get(p_summary.participantId) ph = self.ps_history.get([participant.participantId, 2]) participant = self.participant_dao.get(1) self.assertEquals(ps.lastModified, TIME2) self.assertEquals(ps.hpoId, new_org.hpoId) self.assertEquals(ph.hpoId, insert_org.hpoId) self.assertEquals(ph.organizationId, insert_org.organizationId) self.assertEquals(new_org.hpoId, participant.hpoId) self.assertEquals(new_org.organizationId, participant.organizationId) self.assertIsNone(participant.siteId) self.assertEquals(participant.providerLink, provider_link)
def _setup_data(self): organization_dao = OrganizationDao() site_dao = SiteDao() org_1 = organization_dao.insert( Organization(externalId='ORG_1', displayName='Organization 1', hpoId=PITT_HPO_ID, resourceId='o123456')) organization_dao.insert( Organization(externalId='AARDVARK_ORG', displayName='Aardvarks Rock', hpoId=PITT_HPO_ID, resourceId='o123457')) site_dao.insert( Site(siteName='Site 1', googleGroup='hpo-site-1', mayolinkClientNumber=123456, organizationId=org_1.organizationId, siteStatus=SiteStatus.ACTIVE, enrollingStatus=EnrollingStatus.ACTIVE, launchDate=datetime.datetime(2016, 1, 1), notes='notes', latitude=12.1, longitude=13.1, directions='directions', physicalLocationName='locationName', address1='address1', address2='address2', city='Austin', state='TX', zipCode='78751', phoneNumber='555-555-5555', adminEmails='[email protected], [email protected]', link='http://www.example.com')) site_dao.insert( Site(siteName='Zebras Rock', googleGroup='aaaaaaa', organizationId=org_1.organizationId, enrollingStatus=EnrollingStatus.INACTIVE, siteStatus=SiteStatus.INACTIVE))
def renderFund(self, fund_id): fund = Fund.find(fund_id) profile = Profile.find(fund.prof) organization = Organization.find(profile.org) # these stats are inputs and show up in the top part of the page stats_beta = ['Beta'] stats_controlled = ['Alpha', 'RM', 'RF'] stats_curves = ['c_rate', 'd_rate'] # these stats are calculated and show up after calc is clicked stats_calculated = [ 'growth_rate', 'NAV', 'Unfunded', 'Called', 'Distributed' ] stats = {} x = None for stat_name in stats_beta + stats_controlled + stats_curves + stats_calculated: stat = Stat.find_by_name(stat_name, fund_id, 'db') if not stat == None: stats[stat_name] = { 'y': stat.get_values()[1], # 'color_line': stat.color_line, # 'color_fill': stat.color_fill 'color_line': settings.colors[stat_name]['color_line'], 'color_fill': settings.colors[stat_name]['color_fill'] } if x == None: x = stat.get_values()[0] elif stat_name in stats_controlled + stats_beta + stats_curves: stats[stat_name] = { 'y': [1 for x in range(6)], 'color_line': settings.colors[stat_name]['color_line'], 'color_fill': settings.colors[stat_name]['color_fill'] } if x == None: x = [x for x in range(6)] return { 'fund_name': fund.fund_name, 'fund': fund_id, 'prof_name': profile.prof_name, 'org_name': organization.org_name, 'x': x, 'stats': stats, 'stats_beta': stats_beta, 'stats_curves': stats_curves, 'stats_controlled': stats_controlled, 'stats_calculated': stats_calculated }
def post(self): post_data = self.__check_request(request.form) source = string.digits + string.ascii_letters token = ''.join(map(str, [source[randint(0, len(source) -1) ] for x in range(0,18)])) org = Organization.find(1) user = User(name=post_data['name'], mail_address=post_data['mail_address'], password=post_data['password'], token=token, organization_id=org.id ) user.insert() return jsonify(status=200, message='ok', request=request.form, response={'token':token})
def _setup_unset_enrollment_site(self): site_dao = SiteDao() organization_dao = OrganizationDao() org_2 = organization_dao.insert( Organization(externalId='ORG_2', displayName='Organization 2', hpoId=PITT_HPO_ID)) site_dao.insert( Site(siteName='not enrolling site', googleGroup='not_enrolling_dot_com', organizationId=org_2.organizationId, enrollingStatus=EnrollingStatus.UNSET, siteStatus=SiteStatus.INACTIVE))
def test_participant_different_hpo_does_not_change(self): insert_org = self.organization_dao.insert( Organization(externalId='stark_industries', displayName='ironman', hpoId=PITT_HPO_ID)) self.participant_dao.insert(Participant(participantId=1, biobankId=2)) participant = self.participant_dao.get(1) participant.hpoId = UNSET_HPO_ID self.participant_dao.update(participant) insert_org.hpoId = AZ_HPO_ID self.organization_dao.update(insert_org) new_org = self.organization_dao.get_by_external_id('stark_industries') participant = self.participant_dao.get(1) self.assertNotEqual(new_org.hpoId, participant.hpoId) self.assertEqual(new_org.hpoId, AZ_HPO_ID) self.assertEqual(participant.hpoId, UNSET_HPO_ID)
def _entity_from_row(self, row): hpo = self.hpo_dao.get_by_name( row[ORGANIZATION_AWARDEE_ID_COLUMN].upper()) if hpo is None: logging.warn('Invalid awardee ID %s importing organization %s', row[ORGANIZATION_AWARDEE_ID_COLUMN], row[ORGANIZATION_ORGANIZATION_ID_COLUMN]) self.errors.append( 'Invalid awardee ID {} importing organization {}'.format( row[ORGANIZATION_AWARDEE_ID_COLUMN], row[ORGANIZATION_ORGANIZATION_ID_COLUMN])) return None return Organization( externalId=row[ORGANIZATION_ORGANIZATION_ID_COLUMN].upper(), displayName=row[ORGANIZATION_NAME_COLUMN], hpoId=hpo.hpoId)
def post(self): post_data = self.__check_request(request.form) source = string.digits + string.ascii_letters token = ''.join( map(str, [source[randint(0, len(source) - 1)] for x in range(0, 18)])) org = Organization.find(1) user = User(name=post_data['name'], mail_address=post_data['mail_address'], password=post_data['password'], token=token, organization_id=org.id) user.insert() return jsonify(status=200, message='ok', request=request.form, response={'token': token})
def get_person_organization(self, person_id=None, organization_url=None): url = ("%skp020.asp?KPLFDNR=%s&history=true" % (self.config['scraper']['base_url'], person_id)) logging.info("Getting person organization from %s", url) # Stupid re-try concept because AllRis sometimes misses start < at # tags at first request. try_counter = 0 while True: try: response = self.get_url(url) if not url: return tree = html.fromstring(response.text) memberships = [] person = Person(originalId=person_id) # maps name of type to form name and membership type type_map = { u'Rat der Stadt' : {'mtype' : 'parliament', 'field' : 'PALFDNR'}, u'Parlament' : {'mtype' : 'parliament', 'field' : 'PALFDNR'}, u'Fraktion' : {'mtype' : 'organisation', 'field' : 'FRLFDNR'}, 'Fraktionen': {'mtype' : 'parliament', 'field' : 'FRLFDNR'}, u'Ausschüsse' : {'mtype' : 'organization', 'field' : 'AULFDNR'}, 'Stadtbezirk': {'mtype' : 'parliament', 'field' : 'PALFDNR'}, 'BVV': {'mtype' : 'parliament', 'field' : 'PALFDNR'}, 'Bezirksparlament': {'mtype' : 'parliament', 'field' : 'PALFDNR'}, 'Bezirksverordnetenversammlung': {'mtype' : 'parliament', 'field' : 'PALFDNR'} } # obtain the table with the membership list via a simple state machine mtype = "parliament" field = 'PALFDNR' # for checking if it changes old_group_id = None # for checking if it changes old_group_name = None # might break otherwise group_id = None table = tree.xpath('//*[@id="rismain_raw"]/table[2]') if len(table): table = table[0] for line in table.findall("tr"): if line[0].tag == "th": what = line[0].text.strip() field = None field_list = None if what in type_map: mtype = type_map[what]['mtype'] field = type_map[what]['field'] elif 'Wahlperiode' in what: mtype = 'parliament' # 'FRLFDNR' field_list = ['KPLFDNR', 'AULFDNR'] elif "Auskünfte gemäß BVV" in what: break else: logging.error("Unknown organization type %s " "at person detail page %s", what, person_id) continue else: if "Keine Information" in line.text_content(): # skip because no content is available continue # Empty line = strange stuff comes after this if len(list(line)) < 2: break # first get the name of group group_name = line[1].text_content() organization = Organization(name=group_name) organization.classification = mtype # Now the first col might be a form with more # useful information which will carry through # until we find another one. # With it. we still check the name though. form = line[0].find("form") if form is not None: if field: group_id = int(form.find( "input[@name='%s']" % field).get( "value")) elif field_list: for field in field_list: temp_form = form.find( "input[@name='%s']" % field) if temp_form is not None: group_id = int(temp_form.get( "value")) organization.originalId = group_id # remember it for next loop old_group_id = group_id # remember it for next loop old_group_name = group_name else: # We did not find a form. We assume that the # old group still applies but we nevertheless # check if the groupname is still the same. if old_group_name != group_name: logging.warn("Group name differs but we " "didn't get a form with new " "group id: group name=%s, old " "group name=%s, old group " "id=%s at url %s", group_name, old_group_name, old_group_id, url) organization.originalId = None else: organization.originalId = old_group_id membership = Membership(organization=organization) membership.originalId = (unicode(person_id) + '-' + unicode(group_id)) # TODO: create a list of functions so we can # index them somehow function = line[2].text_content() raw_date = line[3].text_content() # parse the date information if "seit" in raw_date: dparts = raw_date.split() membership.endDate = dparts[-1] elif "Keine" in raw_date or not raw_date.strip(): # no date information available start_date = end_date = None else: dparts = raw_date.split() membership.startDate = dparts[0] membership.endDate = dparts[-1] if organization.originalId is not None: memberships.append(membership) else: logging.warn("Bad organization at %s", url) person.membership = memberships oid = self.db.save_person(person) return else: logging.info("table missing, nothing to do at %s", url) return except AttributeError: if try_counter < 3: logging.info("Try again: Getting person organizations with " "person id %d from %s", person_id, url) try_counter += 1 else: logging.error("Failed getting person organizations with " "person id %d from %s", person_id, url) return
def find_person(self): find_person_url = (self.config['scraper']['base_url'] + 'kp041.asp?template=xyz&selfaction=ws&showAll=true&' 'PALFDNRM=1&kpdatfil=&filtdatum=filter&kpname=&' 'kpsonst=&kpampa=99999999&kpfr=99999999&' 'kpamfr=99999999&kpau=99999999&kpamau=99999999&' 'searchForm=true&search=Suchen') logging.info("Getting person overview from %s", find_person_url) """parse an XML file and return the tree""" parser = etree.XMLParser(recover=True) r = self.get_url(find_person_url) if not r: return xml = r.text.encode('ascii', 'xmlcharrefreplace') tree = etree.fromstring(xml, parser=parser) h = HTMLParser.HTMLParser() # element 0 is the special block # element 1 is the list of persons for node in tree[1].iterchildren(): elem = {} for e in node.iterchildren(): if e.text: elem[e.tag] = h.unescape(e.text) else: elem[e.tag] = '' # now retrieve person details such as organization memberships etc. # we also get the age (but only that, no date of birth) person = Person(originalId=int(elem['kplfdnr'])) if elem['link_kp']: person.originalUrl = elem['link_kp'] # personal information if elem['adtit']: person.title = elem['adtit'] if elem['antext1'] == 'Frau': person.sex = 1 elif elem['antext1'] == 'Herr': person.sex = 2 if elem['advname']: person.firstname = elem['advname'] if elem['adname']: person.lastname = elem['adname'] # address if elem['adstr']: person.address = elem['adstr'] if elem['adhnr']: person.house_number = elem['adhnr'] if elem['adplz']: person.postalcode = elem['adplz'] if elem['adtel']: person.phone = elem['adtel'] # contact if elem['adtel']: person.phone = elem['adtel'] if elem['adtel2']: person.mobile = elem['adtel2'] if elem['adfax']: person.fax = elem['adfax'] if elem['adfax']: person.fax = elem['adfax'] if elem['ademail']: person.email = elem['ademail'] if elem['adwww1']: person.website = elem['adwww1'] person_party = elem['kppartei'] if person_party: if person_party in self.config['scraper']['party_alias']: person_party = self.config['scraper']['party_alias'][ person_party] new_organization = Organization(originalId=person_party, name=person_party, classification='party') original_id = unicode(person.originalId) + '-' + person_party person.membership = [ Membership(originalId=original_id, organization=new_organization) ] if elem['link_kp'] is not None: if hasattr(self, 'person_queue'): self.person_queue.add(person.originalId) else: logging.info("Person %s %s has no link", person.firstname, person.lastname) self.db.save_person(person)
def get_person_organization(self, person_id=None, organization_url=None): url = ("%skp020.asp?KPLFDNR=%s&history=true" % (self.config['scraper']['base_url'], person_id)) logging.info("Getting person organization from %s", url) # Stupid re-try concept because AllRis sometimes misses start < at # tags at first request. try_counter = 0 while True: try: response = self.get_url(url) if not url: return tree = html.fromstring(response.text) memberships = [] person = Person(originalId=person_id) # maps name of type to form name and membership type type_map = { u'Rat der Stadt': { 'mtype': 'parliament', 'field': 'PALFDNR' }, u'Parlament': { 'mtype': 'parliament', 'field': 'PALFDNR' }, u'Fraktion': { 'mtype': 'organisation', 'field': 'FRLFDNR' }, 'Fraktionen': { 'mtype': 'parliament', 'field': 'FRLFDNR' }, u'Ausschüsse': { 'mtype': 'organization', 'field': 'AULFDNR' }, 'Stadtbezirk': { 'mtype': 'parliament', 'field': 'PALFDNR' }, 'BVV': { 'mtype': 'parliament', 'field': 'PALFDNR' }, 'Bezirksparlament': { 'mtype': 'parliament', 'field': 'PALFDNR' }, 'Bezirksverordnetenversammlung': { 'mtype': 'parliament', 'field': 'PALFDNR' } } # obtain the table with the membership list via a simple state machine mtype = "parliament" field = 'PALFDNR' # for checking if it changes old_group_id = None # for checking if it changes old_group_name = None # might break otherwise group_id = None table = tree.xpath('//*[@id="rismain_raw"]/table[2]') if len(table): table = table[0] for line in table.findall("tr"): if line[0].tag == "th": what = line[0].text.strip() field = None field_list = None if what in type_map: mtype = type_map[what]['mtype'] field = type_map[what]['field'] elif 'Wahlperiode' in what: mtype = 'parliament' # 'FRLFDNR' field_list = ['KPLFDNR', 'AULFDNR'] elif "Auskünfte gemäß BVV" in what: break else: logging.error( "Unknown organization type %s " "at person detail page %s", what, person_id) continue else: if "Keine Information" in line.text_content(): # skip because no content is available continue # Empty line = strange stuff comes after this if len(list(line)) < 2: break # first get the name of group group_name = line[1].text_content() organization = Organization(name=group_name) organization.classification = mtype # Now the first col might be a form with more # useful information which will carry through # until we find another one. # With it. we still check the name though. form = line[0].find("form") if form is not None: if field: group_id = int( form.find("input[@name='%s']" % field).get("value")) elif field_list: for field in field_list: temp_form = form.find( "input[@name='%s']" % field) if temp_form is not None: group_id = int( temp_form.get("value")) organization.originalId = group_id # remember it for next loop old_group_id = group_id # remember it for next loop old_group_name = group_name else: # We did not find a form. We assume that the # old group still applies but we nevertheless # check if the groupname is still the same. if old_group_name != group_name: logging.warn( "Group name differs but we " "didn't get a form with new " "group id: group name=%s, old " "group name=%s, old group " "id=%s at url %s", group_name, old_group_name, old_group_id, url) organization.originalId = None else: organization.originalId = old_group_id membership = Membership(organization=organization) membership.originalId = (unicode(person_id) + '-' + unicode(group_id)) # TODO: create a list of functions so we can # index them somehow function = line[2].text_content() raw_date = line[3].text_content() # parse the date information if "seit" in raw_date: dparts = raw_date.split() membership.endDate = dparts[-1] elif "Keine" in raw_date or not raw_date.strip(): # no date information available start_date = end_date = None else: dparts = raw_date.split() membership.startDate = dparts[0] membership.endDate = dparts[-1] if organization.originalId is not None: memberships.append(membership) else: logging.warn("Bad organization at %s", url) person.membership = memberships oid = self.db.save_person(person) return else: logging.info("table missing, nothing to do at %s", url) return except AttributeError: if try_counter < 3: logging.info( "Try again: Getting person organizations with " "person id %d from %s", person_id, url) try_counter += 1 else: logging.error( "Failed getting person organizations with " "person id %d from %s", person_id, url) return
def get_person_organization(self, person_organization_url=None, person_id=None): """ Load committee details for the given detail page URL or numeric ID """ # Read either committee_id or committee_url from the opposite if person_id is not None: person_committee_url = self.urls['PERSON_ORGANIZATION_PRINT_PATTERN'] % (self.config['scraper']['base_url'], person_id) elif person_organization_url is not None: parsed = parse.search(self.urls['PERSON_ORGANIZATION_PRINT_PATTERN'], person_organization_url) person_id = parsed['person_id'] logging.info("Getting person %d organizations from %s", person_id, person_committee_url) person = Person(originalId=person_id) time.sleep(self.config['scraper']['wait_time']) response = self.get_url(person_committee_url) if not response: return # seek(0) is necessary to reset response pointer. response.seek(0) html = response.read() html = html.replace(' ', ' ') parser = etree.HTMLParser() dom = etree.parse(StringIO(html), parser) trs = dom.xpath(self.xpath['PERSON_ORGANIZATION_LINES']) organisations = [] memberships = [] for tr in trs: tds = tr.xpath('.//td') long_info = False if len(tds) == 5: long_info = True if len(tds) == 5 or len(tds) == 2: if tds[0].xpath('.//a'): href = tds[0][0].get('href') href_tmp = href.split('&') # delete __cgrname when it's there if len(href_tmp) == 2: if href_tmp[1][0:10] == '__cgrname=': href = href_tmp[0] parsed = parse.search(self.urls['ORGANIZATION_DETAIL_PARSE_PATTERN'], href) if not parsed: parsed = parse.search(self.urls['ORGANIZATION_DETAIL_PARSE_PATTERN_FULL'], href) if parsed is not None: new_organisation = Organization(originalId=int(parsed['committee_id'])) new_organisation.name = tds[0][0].text else: new_organisation = Organization(originalId=tds[0].text) if new_organisation and long_info: new_membership = Membership() membership_original_id = originalId=unicode(person_id) + '-' + unicode(new_organisation.originalId) if tds[2].text: new_membership.role = tds[2].text if tds[3].text: new_membership.startDate = tds[3].text membership_original_id += '-' + tds[3].text if tds[4].text: new_membership.endDate = tds[4].text membership_original_id += '-' + tds[4].text new_membership.originalId = membership_original_id new_membership.organization = new_organisation memberships.append(new_membership) else: if not new_organisation: logging.error("Bad Table Structure in %s", person_committee_url) if memberships: person.membership = memberships oid = self.db.save_person(person) logging.info("Person %d stored with _id %s", person_id, oid) return
def _make_org(self, **kwargs): org = Organization(**kwargs) self.org_dao.insert(org) return org
def test_schema(self): session = self.database.make_session() hpo = HPO(hpoId=1, name='UNSET', displayName='No organization set', organizationType=OrganizationType.UNSET) code_book = CodeBook(codeBookId=1, created=datetime.datetime.now(), latest=True, name="pmi", system="http://foo/bar", version="v0.1.1") session.add(hpo) session.add(code_book) session.commit() organization = Organization(organizationId=1, externalId='org', displayName='Organization', hpoId=1) session.add(organization) session.commit() site = Site(siteId=1, siteName='site', googleGroup='*****@*****.**', mayolinkClientNumber=12345, organizationId=1) code1 = Code(codeId=1, codeBookId=1, system="a", value="b", shortValue="q", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True, created=datetime.datetime.now()) codeHistory1 = CodeHistory(codeId=1, codeBookId=1, system="a", value="b", shortValue="q", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True, created=datetime.datetime.now()) session.add(site) session.add(code1) session.add(codeHistory1) session.commit() code2 = Code(codeId=2, codeBookId=1, parentId=1, system="a", value="c", display=u"X", topic=u"d", codeType=CodeType.QUESTION, mapped=True, created=datetime.datetime.now()) codeHistory2 = CodeHistory(codeId=2, codeBookId=1, parentId=1, system="a", value="c", display=u"X", topic=u"d", codeType=CodeType.QUESTION, mapped=True, created=datetime.datetime.now()) session.add(code2) session.add(codeHistory2) session.commit() code3 = Code(codeId=3, codeBookId=1, parentId=2, system="a", value="d", display=u"Y", topic=u"d", codeType=CodeType.ANSWER, mapped=False, created=datetime.datetime.now()) codeHistory3 = CodeHistory(codeId=3, codeBookId=1, parentId=2, system="a", value="d", display=u"Y", topic=u"d", codeType=CodeType.ANSWER, mapped=False, created=datetime.datetime.now()) session.add(code3) session.add(codeHistory3) session.commit() session.commit() p = self._participant_with_defaults( participantId=1, version=1, biobankId=2, clientId='*****@*****.**', hpoId=hpo.hpoId, signUpTime=datetime.datetime.now(), lastModified=datetime.datetime.now()) ps = self._participant_summary_with_defaults( participantId=1, biobankId=2, lastModified=datetime.datetime.now(), hpoId=hpo.hpoId, firstName=self.fake.first_name(), middleName=self.fake.first_name(), lastName=self.fake.last_name(), email=self.fake.email(), zipCode='78751', dateOfBirth=datetime.date.today(), genderIdentityId=1, consentForStudyEnrollment=QuestionnaireStatus.SUBMITTED, consentForStudyEnrollmentTime=datetime.datetime.now(), numBaselineSamplesArrived=2) p.participantSummary = ps session.add(p) ph = self._participant_history_with_defaults( participantId=1, biobankId=2, clientId='*****@*****.**', hpoId=hpo.hpoId, signUpTime=datetime.datetime.now(), lastModified=datetime.datetime.now()) session.add(ph) session.commit() session.add( BiobankStoredSample(biobankStoredSampleId='WEB1234542', biobankId=p.biobankId, biobankOrderIdentifier='KIT', test='1UR10', confirmed=datetime.datetime.utcnow())) session.add( BiobankStoredSample( biobankStoredSampleId='WEB99999', # Sample ID must be unique. biobankId=p. biobankId, # Participant ID and test may be duplicated. biobankOrderIdentifier='KIT', test='1UR10', confirmed=datetime.datetime.utcnow())) pm = PhysicalMeasurements(physicalMeasurementsId=1, participantId=1, created=datetime.datetime.now(), resource='blah', final=False, logPosition=LogPosition()) pm2 = PhysicalMeasurements(physicalMeasurementsId=2, participantId=1, created=datetime.datetime.now(), resource='blah', final=True, amendedMeasurementsId=1, logPosition=LogPosition()) session.add(pm) session.add(pm2) session.commit() q1 = Measurement(measurementId=3, physicalMeasurementsId=pm.physicalMeasurementsId, codeSystem='codeSystem', codeValue='codeValue', measurementTime=datetime.datetime.now(), valueCodeSystem='valueCodeSystem', valueCodeValue='value3') session.add(q1) session.commit() m1 = Measurement(measurementId=1, physicalMeasurementsId=pm.physicalMeasurementsId, codeSystem='codeSystem', codeValue='codeValue', measurementTime=datetime.datetime.now(), bodySiteCodeSystem='bodySiteCodeSystem', bodySiteCodeValue='bodySiteCodeValue', valueString='a', valueDecimal=1.2, valueUnit='cm', valueCodeSystem='valueCodeSystem', valueCodeValue='value', valueDateTime=datetime.datetime.now(), qualifierId=q1.measurementId) session.add(m1) session.commit() m2 = Measurement(measurementId=2, physicalMeasurementsId=pm.physicalMeasurementsId, codeSystem='codeSystem', codeValue='codeValue', measurementTime=datetime.datetime.now(), valueCodeSystem='valueCodeSystem', valueCodeValue='value2', parentId=m1.measurementId, qualifierId=q1.measurementId) session.add(m2) session.commit() q = Questionnaire(questionnaireId=1, version=1, created=datetime.datetime.now(), lastModified=datetime.datetime.now(), resource='what?') qh = QuestionnaireHistory(questionnaireId=1, version=1, created=datetime.datetime.now(), lastModified=datetime.datetime.now(), resource='what?') qh.questions.append( QuestionnaireQuestion(questionnaireQuestionId=1, questionnaireId=1, questionnaireVersion=1, linkId="1.2.3", codeId=2, repeats=True)) qh.concepts.append( QuestionnaireConcept(questionnaireConceptId=1, questionnaireId=1, questionnaireVersion=1, codeId=1)) session.add(q) session.add(qh) session.commit() qr = QuestionnaireResponse(questionnaireResponseId=1, questionnaireId=1, questionnaireVersion=1, participantId=1, created=datetime.datetime.now(), resource='blah') qr.answers.append( QuestionnaireResponseAnswer(questionnaireResponseAnswerId=1, questionnaireResponseId=1, questionId=1, endTime=datetime.datetime.now(), valueSystem='a', valueCodeId=3, valueDecimal=123, valueString=self.fake.first_name(), valueDate=datetime.date.today())) session.add(qr) session.commit() mv = MetricsVersion(metricsVersionId=1, inProgress=False, complete=True, date=datetime.datetime.utcnow(), dataVersion=1) session.add(mv) session.commit() mb = MetricsBucket(metricsVersionId=1, date=datetime.date.today(), hpoId='PITT', metrics='blah') session.add(mb) session.commit()