def convertSourceItems(self, convOptions, conn=None):
"""Primary run function to process the contents of the raw source
table and convert them into equivalent patient_item, clinical_item, and clinical_item_category entries.
Should look for redundancies after the fact to catch repeated conversions.
startDate - If provided, only return items whose ordering_date is on or after that date.
endDate - If provided, only return items whose ordering_date is before that date.
"""
log.info("Conversion for items dated %s to %s" %
(convOptions.startDate, convOptions.endDate))
progress = ProgressDots()
extConn = conn is not None
if not extConn:
conn = self.connFactory.connection()
try:
# Next round for medications directly from order_med table not addressed in medmix TODO (nodir) seems like an unrelated comment?
category = self.categoryFromSourceItem(conn)
for sourceItem in self.querySourceItems(convOptions):
log.debug('sourceItem: {}'.format(sourceItem))
self.convertSourceItem(category, sourceItem, conn=conn)
progress.Update()
finally:
conn.close()
progress.PrintStatus()
def convertSourceItems(self, convOptions):
"""Primary run function to process the contents of the order_med
table and convert them into equivalent patient_item, clinical_item, and clinical_item_category entries.
Should look for redundancies after the fact to catch repeatEd conversions.
startDate - If provided, only return items whose order_time_jittered is on or after that date.
endDate - If provided, only return items whose order_time_jittered is before that date.
"""
log.info("Conversion for items dated {} to {}".format(
convOptions.startDate, convOptions.endDate))
progress = ProgressDots()
conn = self.connFactory.connection()
try:
# Load up the medication mapping table to facilitate subsequent conversions
rxcuiDataByMedId = self.loadRXCUIData()
# Next round for medications directly from order_med table not addressed in medmix
for sourceItem in self.querySourceItems(rxcuiDataByMedId,
convOptions,
progress=progress,
conn=conn):
self.convertSourceItem(sourceItem, conn=conn)
progress.Update()
finally:
conn.close()
progress.PrintStatus()
def convertSourceItems(self, startDate=None, endDate=None):
"""Primary run function to process the contents of the source table
and convert them into equivalent patient_item, clinical_item, and clinical_item_category entries.
Should look for redundancies to avoid repeating conversion.
startDate - If provided, only return items whose noted_date is on or after that date.
endDate - If provided, only return items whose noted_date is before that date.
"""
log.info("Conversion for items dated %s to %s" % (startDate, endDate));
progress = ProgressDots();
conn = self.connFactory.connection();
try:
for sourceItem in self.querySourceItems(startDate, endDate, progress=progress, conn=conn):
self.convertSourceItem(sourceItem, conn=conn);
finally:
conn.close();
progress.PrintStatus();
def convertSourceItems(self, convOptions):
"""Primary run function to process the contents of the stride_order_med
table and convert them into equivalent patient_item, clinical_item, and clinical_item_category entries.
Should look for redundancies after the fact to catch repeatEd conversions.
startDate - If provided, only return items whose ordering_date is on or after that date.
endDate - If provided, only return items whose ordering_date is before that date.
"""
log.info("Conversion for items dated %s to %s" %
(convOptions.startDate, convOptions.endDate))
progress = ProgressDots()
conn = self.connFactory.connection()
try:
# Load up the medication mapping table to facilitate subsequent conversions
rxcuiDataByMedId = self.loadRXCUIData(conn=conn)
# Keep track of which order meds have already been converted based on mixture components (don't repeat for the aggregate order then)
# Can be a lot to store in local memory for large conversions, so may need to batch smaller sub-processes
convertedOrderMedIds = set()
# First round for medication combinations that must be extracted from order_medmixinfo table
for sourceItem in self.queryMixSourceItems(rxcuiDataByMedId,
convOptions,
progress=progress,
conn=conn):
self.convertSourceItem(sourceItem, conn=conn)
convertedOrderMedIds.add(sourceItem["order_med_id"])
progress.Update()
# Next round for medications directly from order_med table not addressed in medmix
for sourceItem in self.querySourceItems(rxcuiDataByMedId,
convOptions,
progress=progress,
conn=conn):
if sourceItem[
"order_med_id"] not in convertedOrderMedIds: # Don't repeat conversion if mixture components already addressed
self.convertSourceItem(sourceItem, conn=conn)
progress.Update()
finally:
conn.close()
progress.PrintStatus()
def convertSourceItems(self, convOptions):
"""Primary run function to process the contents of the raw source
table and convert them into equivalent patient_item, clinical_item, and clinical_item_category entries.
Should look for redundancies after the fact to catch repeated conversions.
startDate - If provided, only return items whose ordering_date is on or after that date.
endDate - If provided, only return items whose ordering_date is before that date.
"""
log.info("Conversion for items dated %s to %s" % (convOptions.startDate, convOptions.endDate));
progress = ProgressDots();
conn = self.connFactory.connection();
try:
# Next round for medications directly from order_med table not addressed in medmix
for sourceItem in self.querySourceItems(convOptions, progress=progress, conn=conn):
self.convertSourceItem(sourceItem, conn=conn);
progress.Update();
finally:
conn.close();
progress.PrintStatus();
def syncTable(sourceConn,
targetConn,
syncTableName,
rowIDStrSet=None,
formatter=None):
if formatter is None:
idCol = DBUtil.defaultIDColumn(syncTableName)
idQuery = "select %s from %s" % (idCol, syncTableName)
# Collect all of the IDs known in the target database and store in memory for rapid lookup
print("Querying for IDs from Target Database", file=sys.stderr)
targetIdTable = DBUtil.execute(idQuery, conn=targetConn)
targetIdSet = set()
for row in targetIdTable:
targetId = row[0]
targetIdSet.add(targetId)
# Query data out of the source table, but do it by a cursor so we can stream through large data tables
print("Querying for Source Data", file=sys.stderr)
dataQuery = "select * from %s" % (syncTableName)
sourceCursor = sourceConn.cursor()
sourceCursor.execute(dataQuery)
colNames = DBUtil.columnNamesFromCursor(sourceCursor)
targetCursor = targetConn.cursor()
insertQuery = None
updateQuery = None
progress = ProgressDots()
row = sourceCursor.fetchone()
while row is not None:
dataModel = RowItemModel(row, colNames)
if rowIDStrSet is None or str(dataModel[idCol]) in rowIDStrSet:
if rowIDStrSet is not None:
print("Syncing record: %s" % dataModel[idCol],
file=sys.stderr)
if dataModel[idCol] not in targetIdSet:
# Row does not yet exist in target database, need to insert it
if insertQuery is None:
insertQuery = DBUtil.buildInsertQuery(
syncTableName, list(dataModel.keys()))
insertParams = list(dataModel.values())
targetCursor.execute(insertQuery, insertParams)
else:
# Row already exists in target database, just update values
if updateQuery is None:
updateQuery = DBUtil.buildUpdateQuery(
syncTableName, list(dataModel.keys()))
updateParams = []
updateParams.extend(list(dataModel.values()))
updateParams.append(dataModel[idCol])
targetCursor.execute(updateQuery, updateParams)
if progress.GetCounts() % progress.big == 0:
targetConn.commit()
row = sourceCursor.fetchone()
progress.Update()
progress.PrintStatus()
targetConn.commit()
else:
##Do something with thr formatter
##Set up the formatter
theFormatter = formatter(syncTableName,
targetConn,
includeColumnNames=True,
autoCommit=True)
#Call DB execute
res = DBUtil.execute("select * from %s" % syncTableName,
includeColumnNames=True,
conn=sourceConn,
formatter=theFormatter)
