def test_basics(self):
I2B2Core.update_date = datetime(2017, 1, 3)
with self.sourcesystem_cd():
I2B2Core.sourcesystem_cd = self._sourcesystem_cd
x = PatientDimension(
12345,
VitalStatusCd(VitalStatusCd.bd_unknown,
VitalStatusCd.dd_unknown))
self.assertEqual(
'patient_num\tvital_status_cd\tbirth_date\tdeath_date\tsex_cd\tage_in_years_num\t'
'language_cd\trace_cd\tmarital_status_cd\treligion_cd\tzip_cd\tstatecityzip_path\t'
'income_cd\tpatient_blob\tupdate_date\tdownload_date\timport_date\t'
'sourcesystem_cd\tupload_id', heading(x))
self.assertEqual(
OrderedDict([('patient_num', 12345), ('vital_status_cd', 'UL'),
('birth_date', None), ('death_date', None),
('sex_cd', None), ('age_in_years_num', None),
('language_cd', None), ('race_cd', None),
('marital_status_cd', None),
('religion_cd', None), ('zip_cd', None),
('statecityzip_path', None), ('income_cd', None),
('patient_blob', None),
('update_date', datetime(2017, 1, 3, 0, 0)),
('download_date', datetime(2017, 1, 3, 0, 0)),
('import_date', datetime(2017, 1, 3, 0, 0)),
('sourcesystem_cd', self._sourcesystem_cd),
('upload_id', None)]), as_dict(x))
x.birth_date = datetime(1955, 8, 10)
x.death_date = datetime(2017, 9, 11)
x.birth = VitalStatusCd.bd_day
x.death = VitalStatusCd.dd_deceased
x.sex_cd = "M"
x.age_in_years = 62
x.language_cd = 'english'
x.race_cd = 'white'
x.marital_status_cd = 'married'
x.religion_cd = 'atheist'
x.zip_cd = "55901-0138"
x.statecityzip_path = 'Zip codes\\Minnesota\\Rochester\\55901\\'
x.income_cd = "Medium"
x.patient_blob = "<div>some text</div>"
self.assertEqual(
OrderedDict([('patient_num', 12345), ('vital_status_cd', 'UL'),
('birth_date', datetime(1955, 8, 10, 0, 0)),
('death_date', datetime(2017, 9, 11, 0, 0)),
('sex_cd', 'M'), ('age_in_years_num', None),
('language_cd', 'english'), ('race_cd', 'white'),
('marital_status_cd', 'married'),
('religion_cd', 'atheist'),
('zip_cd', '55901-0138'),
('statecityzip_path',
'Zip codes\\Minnesota\\Rochester\\55901\\'),
('income_cd', 'Medium'),
('patient_blob', '<div>some text</div>'),
('update_date', datetime(2017, 1, 3, 0, 0)),
('download_date', datetime(2017, 1, 3, 0, 0)),
('import_date', datetime(2017, 1, 3, 0, 0)),
('sourcesystem_cd', self._sourcesystem_cd),
('upload_id', None)]), as_dict(x))
def _add_or_update_records(cls, conn: Connection, table: Table,
records: List["I2B2CoreWithUploadId"]) -> Tuple[int, int]:
"""Add or update the supplied table as needed to reflect the contents of records
:param table: i2b2 sql connection
:param records: records to apply
:return: number of records added / modified
"""
num_updates = 0
num_inserts = 0
inserts = []
# Iterate over the records doing updates
# Note: This is slow as molasses - definitely not optimal for batch work, but hopefully we'll be dealing with
# thousands to tens of thousands of records. May want to move to ORM model if this gets to be an issue
for record in records:
keys = [(table.c[k] == getattr(record, k)) for k in cls.key_fields]
key_filter = I2B2CoreWithUploadId._nested_fcn(and_, keys)
rec_exists = conn.execute(select([table.c.upload_id]).where(key_filter)).rowcount
if rec_exists:
known_values = {k: v for k, v in as_dict(record).items()
if v is not None and k not in cls._no_update_fields and
k not in cls.key_fields}
vals = [table.c[k] != v for k, v in known_values.items()]
val_filter = I2B2CoreWithUploadId._nested_fcn(or_, vals)
known_values['update_date'] = record.update_date
upd = update(table).where(and_(key_filter, val_filter)).values(known_values)
num_updates += conn.execute(upd).rowcount
else:
inserts.append(as_dict(record))
if inserts:
if cls._check_dups:
dups = cls._check_for_dups(inserts)
nprints = 0
if dups:
print("{} duplicate records encountered".format(len(dups)))
for k, vals in dups.items():
if len(vals) == 2 and vals[0] == vals[1]:
inserts.remove(vals[1])
else:
if nprints < 20:
print("Key: {} has a non-identical dup".format(k))
elif nprints == 20:
print(".... more ...")
nprints += 1
for v in vals[1:]:
inserts.remove(v)
# TODO: refactor this to load on a per-resource basis. Temporary fix
for insert in ListChunker(inserts, 500):
num_inserts += conn.execute(table.insert(), insert).rowcount
return num_inserts, num_updates
def test_general(self):
from i2b2model.metadata.i2b2ontologyquery import Query
TableAccess.c_entry_date = datetime(2018, 3, 24, 11, 17)
q = Query('ITCP_EVENTS', 'concept_cd', False, 'C_FULLNAME', '=',
'\\SCT\\276746005')
ta = TableAccess('SCT_ENV_EVENT', '\\SCT\\276746005\\', q, 2,
'Environmental event (event)')
self.assertEqual(
OrderedDict([('c_table_cd', 'SCT_ENV_EVENT'),
('c_table_name', 'custom_meta'),
('c_protected_access', 'N'), ('c_hlevel', 2),
('c_fullname', '\\SCT\\276746005\\'),
('c_name', 'Environmental event (event)'),
('c_synonym_cd', 'N'), ('c_visualattributes', 'CA '),
('c_totalnum', None), ('c_basecode', None),
('c_metadataxml', None),
('c_facttablecolumn', 'concept_cd'),
('c_dimtablename', 'ITCP_EVENTS'),
('c_columnname', 'C_FULLNAME'),
('c_columndatatype', 'T'), ('c_operator', '='),
('c_dimcode', '\\SCT\\276746005'),
('c_comment', None),
('c_tooltip', 'Environmental event (event)'),
('c_entry_date', datetime(2018, 3, 24, 11, 17)),
('c_change_date', None), ('c_status_cd', None),
('valuetype_cd', None)]), as_dict(ta))
def test_fhir(self):
TableAccess.c_entry_date = datetime(2018, 3, 24, 11, 12)
ta = TableAccess('FHIR')
self.assertEqual((
'c_table_cd\tc_table_name\tc_protected_access\tc_hlevel\tc_fullname\tc_name\t'
'c_synonym_cd\tc_visualattributes\tc_totalnum\tc_basecode\tc_metadataxml\t'
'c_facttablecolumn\tc_dimtablename\tc_columnname\tc_columndatatype\tc_operator\t'
'c_dimcode\tc_comment\tc_tooltip\tc_entry_date\tc_change_date\tc_status_cd\t'
'valuetype_cd'), heading(ta))
self.assertEqual(
OrderedDict([('c_table_cd', 'FHIR'),
('c_table_name', DEFAULT_ONTOLOGY_TABLE),
('c_protected_access', 'N'), ('c_hlevel', 1),
('c_fullname', '\\FHIR\\'),
('c_name', 'FHIR Resources'), ('c_synonym_cd', 'N'),
('c_visualattributes', 'CA '), ('c_totalnum', None),
('c_basecode', None), ('c_metadataxml', None),
('c_facttablecolumn', 'concept_cd'),
('c_dimtablename', 'concept_dimension'),
('c_columnname', 'concept_path'),
('c_columndatatype', 'T'), ('c_operator', '='),
('c_dimcode', '\\FHIR\\'), ('c_comment', None),
('c_tooltip', 'FHIR Resources'),
('c_entry_date', datetime(2018, 3, 24, 11, 12)),
('c_change_date', None), ('c_status_cd', None),
('valuetype_cd', None)]), as_dict(ta))
def test_basics(self):
from i2b2model.data.i2b2observationfact import ObservationFact, ObservationFactKey
I2B2Core.update_date = datetime(2017, 2, 19, 12, 33)
with self.sourcesystem_cd():
I2B2Core.sourcesystem_cd = self._sourcesystem_cd
ofk = ObservationFactKey(12345, 23456, 'provider',
datetime(2017, 5, 23, 11, 17))
x = ObservationFact(ofk, 'fhir:concept')
self.assertEqual(
'encounter_num\tpatient_num\tconcept_cd\tprovider_id\tstart_date\tmodifier_cd\t'
'instance_num\tvaltype_cd\ttval_char\tnval_num\tvalueflag_cd\tquantity_num\tunits_cd\t'
'end_date\tlocation_cd\tobservation_blob\tconfidence_num\tupdate_date\tdownload_date\t'
'import_date\tsourcesystem_cd\tupload_id', heading(x))
self.assertEqual(
OrderedDict([('encounter_num', 23456), ('patient_num', 12345),
('concept_cd', 'fhir:concept'),
('provider_id', 'provider'),
('start_date', datetime(2017, 5, 23, 11, 17)),
('modifier_cd', '@'), ('instance_num', 0),
('valtype_cd', '@'), ('tval_char', None),
('nval_num', None), ('valueflag_cd', None),
('quantity_num', None), ('units_cd', None),
('end_date', None), ('location_cd', None),
('observation_blob', None),
('confidence_num', None),
('update_date', datetime(2017, 2, 19, 12, 33)),
('download_date', datetime(2017, 2, 19, 12, 33)),
('import_date', datetime(2017, 2, 19, 12, 33)),
('sourcesystem_cd', self._sourcesystem_cd),
('upload_id', None)]), as_dict(x))
def add_or_update_record(self, tables: I2B2Tables) -> Tuple[int, int]:
conn = tables.ont_connection
table = tables.schemes
numins, numupd = 0, 0
rslt = list(
conn.execute(table.select().where(table.c.c_key == self.c_key)))
if rslt:
for row in rslt:
if not self._matches(row, as_dict(self), ['']):
conn.execute(table.update().where(
table.c.c_key == self.c_key).values(as_dict(self)))
numupd = 1
else:
conn.execute(table.insert().values(as_dict(self)))
numins = 1
return numins, numupd
def test_basics(self):
from i2b2model.data.i2b2visitdimension import VisitDimension, ActiveStatusCd
I2B2Core.update_date = datetime(2017, 1, 3)
with self.sourcesystem_cd():
I2B2CoreWithUploadId.upload_id = self._upload_id
I2B2Core.sourcesystem_cd = self._sourcesystem_cd
x = VisitDimension(500001, 10000017, ActiveStatusCd(ActiveStatusCd.sd_day,
ActiveStatusCd.ed_ongoing), datetime(2007, 10, 4))
self.assertEqual(OrderedDict([
('encounter_num', 500001),
('patient_num', 10000017),
('active_status_cd', 'OD'),
('start_date', datetime(2007, 10, 4, 0, 0)),
('end_date', None),
('inout_cd', None),
('location_cd', None),
('location_path', None),
('length_of_stay', None),
('visit_blob', None),
('update_date', datetime(2017, 1, 3, 0, 0)),
('download_date', datetime(2017, 1, 3, 0, 0)),
('import_date', datetime(2017, 1, 3, 0, 0)),
('sourcesystem_cd', self._sourcesystem_cd),
('upload_id', self._upload_id)]), as_dict(x))
def add_or_update_record(self, tables: I2B2Tables) -> Tuple[int, int]:
conn = tables.ont_connection
table = tables.schemes
numins, numupd = 0, 0
rslt = list(conn.execute(table.select().where(table.c.c_table_cd == self.c_table_cd)))
if rslt:
for row in rslt:
if row[1] != self.c_name or row[2] != self.c_description:
conn.execute(table.update().where(table.c.c_key == self.c_key).values(as_dict(self)))
numupd = 1
else:
conn.execute(table.insert().values(as_dict(self)))
numins = 1
return numins, numupd
def test_settings(self):
clear(I2B2Core)
I2B2Core.sourcesystem_cd = "abcd"
I2B2Core.update_date = datetime(2014, 7, 31)
I2B2Core.download_date = datetime.now
rtn = I2B2Core()
rtnf = as_dict(rtn)
self.assertEqual(str(rtnf['update_date']), '2014-07-31 00:00:00')
self.assertAlmostNow(rtn.download_date)
self.assertEqual(rtn.update_date, rtn.import_date)
self.assertEqual(rtnf['sourcesystem_cd'], 'abcd')
self.assertEqual(
'update_date\tdownload_date\timport_date\tsourcesystem_cd',
heading(rtn))
def test_basics(self):
from i2b2model.metadata.i2b2conceptdimension import ConceptDimension
clear(ConceptDimension)
I2B2Core.download_date = datetime(2017, 5, 25)
I2B2Core.sourcesystem_cd = "TEST_SS"
I2B2Core.import_date = datetime(2017, 5, 25)
cd = ConceptDimension('TEST', 'root', 'Root test concept',
['L1', 'L2', 'root'], '\\TEST\\')
self.assertAlmostNow(cd.update_date)
I2B2Core.update_date = datetime(2001, 12, 1)
expected = OrderedDict([('concept_path', '\\TEST\\L1\\L2\\root\\'),
('concept_cd', 'TEST:root'),
('name_char', 'TEST Root test concept'),
('concept_blob', ''),
('update_date', datetime(2001, 12, 1, 0, 0)),
('download_date', datetime(2017, 5, 25, 0, 0)),
('import_date', datetime(2017, 5, 25, 0, 0)),
('sourcesystem_cd', 'TEST_SS'),
('upload_id', None)])
self.assertEqual(expected, as_dict(cd))
# Note - balance is actually a modifier. This is strictly an example
cd = ConceptDimension('TEST', 'root', 'Root balance test concept',
['L1', 'L2', 'balance'], '\\TEST\\')
expected = OrderedDict([
('concept_path', '\\TEST\\L1\\L2\\balance\\root\\'),
('concept_cd', 'TEST:root'),
('name_char', 'TEST Root balance test concept'),
('concept_blob', ''), ('update_date', datetime(2001, 12, 1, 0, 0)),
('download_date', datetime(2017, 5, 25, 0, 0)),
('import_date', datetime(2017, 5, 25, 0, 0)),
('sourcesystem_cd', 'TEST_SS'), ('upload_id', None)
])
self.assertEqual(expected, as_dict(cd))
def test_create(self):
self.assertEqual("c_key\tc_name\tc_description", heading(self.rec))
self.assertEqual('TestCodeSystem\tA test coding system\tUsed for unit tests - you should never see this',
row(self.rec))
self.assertEqual(OrderedDict([
('c_key', 'TestCodeSystem'),
('c_name', 'A test coding system'),
('c_description',
'Used for unit tests - you should never see this')]), as_dict(self.rec))
self.assertEqual((1, 0), self.rec.add_or_update_record(self.opts.tables))
self.assertEqual((0, 0), self.rec.add_or_update_record(self.opts.tables))
self.rec.c_description = "Used for tests"
self.assertEqual((0, 1), self.rec.add_or_update_record(self.opts.tables))
self.assertEqual(1, self.rec.del_record(self.opts.tables))
self.assertEqual(0, self.rec.del_record(self.opts.tables))
def test_basics(self):
from i2b2model.metadata.i2b2modifierdimension import ModifierDimension
I2B2Core.download_date = datetime(2017, 5, 25)
I2B2Core.sourcesystem_cd = "MOD_TEST"
I2B2Core.import_date = datetime(2017, 5, 25)
md = ModifierDimension('MODTEST', 'baboon', 'Wild baboons',
['Earth', 'Africa', 'Zimbabwai'])
self.assertAlmostNow(md.update_date)
I2B2Core.update_date = datetime(2001, 12, 1)
expected = OrderedDict([('modifier_path',
'\\Earth\\Africa\\Zimbabwai\\baboon\\'),
('modifier_cd', 'MODTEST:baboon'),
('name_char', 'MODTEST Wild baboons'),
('modifier_blob', ''),
('update_date', datetime(2001, 12, 1, 0, 0)),
('download_date', datetime(2017, 5, 25, 0, 0)),
('import_date', datetime(2017, 5, 25, 0, 0)),
('sourcesystem_cd', 'MOD_TEST'),
('upload_id', None)])
self.assertEqual(expected, as_dict(md))
def test_defaults(self):
rtn = I2B2Core()
rtnf = as_dict(rtn)
self.assertAlmostNow(rtn.update_date)
self.assertDatesAlmostEqual(rtn.update_date, str(rtnf['update_date']))
self.assertAlmostNow(rtn.download_date)
self.assertDatesAlmostEqual(rtn.download_date,
str(rtnf['download_date']))
self.assertAlmostNow(rtn.import_date)
self.assertDatesAlmostEqual(rtn.import_date, str(rtnf['import_date']))
self.assertEqual('Unspecified', rtn.sourcesystem_cd)
self.assertEqual(rtn.sourcesystem_cd, rtnf['sourcesystem_cd'])
self.assertEqual(
['update_date', 'download_date', 'import_date', 'sourcesystem_cd'],
list(rtnf.keys()))
rtn = I2B2Core()
I2B2Core.download_date = datetime(2009, 1, 1, 12, 0)
I2B2Core.sourcesystem_cd = "MASTER"
I2B2Core.import_date = datetime(2011, 1, 1, 12, 0)
I2B2Core.update_date = datetime.now() + timedelta(hours=2)
self.assertEqual('2009-01-01 12:00:00', str(rtn.download_date))
self.assertEqual('2011-01-01 12:00:00', str(rtn.import_date))
self.assertEqual('MASTER', rtn.sourcesystem_cd)
self.assertDatesAlmostEqual(rtn.update_date,
str(datetime.now() + timedelta(hours=2)))
clear(I2B2Core)
I2B2CoreWithUploadId.upload_id = 1777439
rtn = I2B2CoreWithUploadId()
if rtn.sourcesystem_cd != 'Unspecified':
print("Caught it")
self.assertEqual('Unspecified', rtn.sourcesystem_cd)
self.assertEqual(1777439, rtn.upload_id)
self.assertEqual(
'update_date\tdownload_date\timport_date\tsourcesystem_cd\tupload_id',
heading(rtn))
rtn = I2B2Core()
with self.assertRaises(AttributeError):
_ = rtn.upload_id
def test_patient_mapping(self):
from i2b2model.data.i2b2patientmapping import PatientMapping
from i2b2model.data.i2b2patientmapping import PatientIDEStatus
I2B2Core.update_date = datetime(2017, 5, 25)
with self.sourcesystem_cd():
I2B2Core.sourcesystem_cd = self._sourcesystem_cd
pm = PatientMapping(10000001, "p123", PatientIDEStatus.active,
"http://hl7.org/fhir/", "fhir")
I2B2CoreWithUploadId.upload_id = 17443
self.assertEqual(
OrderedDict([('patient_ide', 'p123'),
('patient_ide_source', 'http://hl7.org/fhir/'),
('patient_num', 10000001),
('patient_ide_status', 'A'),
('project_id', 'fhir'),
('update_date', datetime(2017, 5, 25, 0, 0)),
('download_date', datetime(2017, 5, 25, 0, 0)),
('import_date', datetime(2017, 5, 25, 0, 0)),
('sourcesystem_cd', self._sourcesystem_cd),
('upload_id', 17443)]), as_dict(pm))
def test_basics(self):
from i2b2model.metadata.i2b2ontology import OntologyEntry
from i2b2model.metadata.i2b2ontologyquery import ConceptQuery
I2B2Core.download_date = datetime.datetime(2017, 5, 25)
I2B2Core.sourcesystem_cd = "TEST"
I2B2Core.import_date = datetime.datetime(2017, 5, 25)
I2B2Core.update_date = datetime.datetime(2001, 12, 1)
ontrec = OntologyEntry('\\X\\Y\\Z\\', ConceptQuery('\\X\\Y\\Z\\'),
None, "17400008")
self.assertEqual(
'c_hlevel\tc_fullname\tc_name\tc_synonym_cd\tc_visualattributes\t'
'c_totalnum\tc_basecode\tc_metadataxml\tc_facttablecolumn\tc_tablename\t'
'c_columnname\tc_columndatatype\tc_operator\tc_dimcode\tc_comment\t'
'c_tooltip\tm_applied_path\tupdate_date\tdownload_date\t'
'import_date\tsourcesystem_cd\tvaluetype_cd\tm_exclusion_cd\tc_path\tc_symbol',
heading(ontrec))
# Note that hierarchy level is zero based
self.assertEqual(
OrderedDict([('c_hlevel', 2), ('c_fullname', '\\X\\Y\\Z\\'),
('c_name', 'Z'), ('c_synonym_cd', 'N'),
('c_visualattributes', 'FAE'), ('c_totalnum', None),
('c_basecode', '17400008'), ('c_metadataxml', None),
('c_facttablecolumn', 'concept_cd'),
('c_tablename', 'concept_dimension'),
('c_columnname', 'concept_path'),
('c_columndatatype', 'T'), ('c_operator', '='),
('c_dimcode', '\\X\\Y\\Z\\'), ('c_comment', None),
('c_tooltip', None), ('m_applied_path', '@'),
('update_date', datetime.datetime(2001, 12, 1, 0, 0)),
('download_date',
datetime.datetime(2017, 5, 25, 0, 0)),
('import_date', datetime.datetime(2017, 5, 25, 0, 0)),
('sourcesystem_cd', 'TEST'), ('valuetype_cd', None),
('m_exclusion_cd', None), ('c_path', None),
('c_symbol', None)]), as_dict(ontrec))
def test_reify(self):
class SpecialProp1:
def __init__(self, parts) -> None:
self.parts = parts
def reify(self):
return '-'.join(str(s)
for s in self.parts) if self.parts else None
class SpecialProp2:
def __init__(self, *parts) -> None:
self.parts = parts
def reify(self):
return sum(int(p) for p in self.parts) if self.parts else None
class R1(DynProps):
sp1: Local[SpecialProp1]
sp2: Local[SpecialProp1]
sp3: Local[SpecialProp2]
sp4: Local[SpecialProp2]
r = R1()
r.sp1 = SpecialProp1(['a', 17, None])
r.sp2 = SpecialProp1([])
r.sp3 = SpecialProp2(17, -3, 100101)
r.sp4 = SpecialProp2()
self.assertEqual("a-17-None\t\t100115\t", row(r))
self.assertEqual(
OrderedDict([('sp1', 'a-17-None'), ('sp2', None), ('sp3', 100115),
('sp4', None)]), as_dict(r))
r.sp1_.parts.append("Alpha")
self.assertEqual('a-17-None-Alpha', r.sp1)
self.assertTrue(isinstance(r.sp1_, SpecialProp1))
self.assertTrue(isinstance(r.sp1, str))