def convertSourceItems(self, convOptions, conn=None): """Primary run function to process the contents of the raw source table and convert them into equivalent patient_item, clinical_item, and clinical_item_category entries. Should look for redundancies after the fact to catch repeated conversions. startDate - If provided, only return items whose ordering_date is on or after that date. endDate - If provided, only return items whose ordering_date is before that date. """ log.info("Conversion for items dated %s to %s" % (convOptions.startDate, convOptions.endDate)) progress = ProgressDots() extConn = conn is not None if not extConn: conn = self.connFactory.connection() try: # Next round for medications directly from order_med table not addressed in medmix TODO (nodir) seems like an unrelated comment? category = self.categoryFromSourceItem(conn) for sourceItem in self.querySourceItems(convOptions): log.debug('sourceItem: {}'.format(sourceItem)) self.convertSourceItem(category, sourceItem, conn=conn) progress.Update() finally: conn.close() progress.PrintStatus()
def convertSourceItems(self, convOptions): """Primary run function to process the contents of the order_med table and convert them into equivalent patient_item, clinical_item, and clinical_item_category entries. Should look for redundancies after the fact to catch repeatEd conversions. startDate - If provided, only return items whose order_time_jittered is on or after that date. endDate - If provided, only return items whose order_time_jittered is before that date. """ log.info("Conversion for items dated {} to {}".format( convOptions.startDate, convOptions.endDate)) progress = ProgressDots() conn = self.connFactory.connection() try: # Load up the medication mapping table to facilitate subsequent conversions rxcuiDataByMedId = self.loadRXCUIData() # Next round for medications directly from order_med table not addressed in medmix for sourceItem in self.querySourceItems(rxcuiDataByMedId, convOptions, progress=progress, conn=conn): self.convertSourceItem(sourceItem, conn=conn) progress.Update() finally: conn.close() progress.PrintStatus()
def convertSourceItems(self, startDate=None, endDate=None): """Primary run function to process the contents of the source table and convert them into equivalent patient_item, clinical_item, and clinical_item_category entries. Should look for redundancies to avoid repeating conversion. startDate - If provided, only return items whose noted_date is on or after that date. endDate - If provided, only return items whose noted_date is before that date. """ log.info("Conversion for items dated %s to %s" % (startDate, endDate)); progress = ProgressDots(); conn = self.connFactory.connection(); try: for sourceItem in self.querySourceItems(startDate, endDate, progress=progress, conn=conn): self.convertSourceItem(sourceItem, conn=conn); finally: conn.close(); progress.PrintStatus();
def convertSourceItems(self, convOptions): """Primary run function to process the contents of the stride_order_med table and convert them into equivalent patient_item, clinical_item, and clinical_item_category entries. Should look for redundancies after the fact to catch repeatEd conversions. startDate - If provided, only return items whose ordering_date is on or after that date. endDate - If provided, only return items whose ordering_date is before that date. """ log.info("Conversion for items dated %s to %s" % (convOptions.startDate, convOptions.endDate)) progress = ProgressDots() conn = self.connFactory.connection() try: # Load up the medication mapping table to facilitate subsequent conversions rxcuiDataByMedId = self.loadRXCUIData(conn=conn) # Keep track of which order meds have already been converted based on mixture components (don't repeat for the aggregate order then) # Can be a lot to store in local memory for large conversions, so may need to batch smaller sub-processes convertedOrderMedIds = set() # First round for medication combinations that must be extracted from order_medmixinfo table for sourceItem in self.queryMixSourceItems(rxcuiDataByMedId, convOptions, progress=progress, conn=conn): self.convertSourceItem(sourceItem, conn=conn) convertedOrderMedIds.add(sourceItem["order_med_id"]) progress.Update() # Next round for medications directly from order_med table not addressed in medmix for sourceItem in self.querySourceItems(rxcuiDataByMedId, convOptions, progress=progress, conn=conn): if sourceItem[ "order_med_id"] not in convertedOrderMedIds: # Don't repeat conversion if mixture components already addressed self.convertSourceItem(sourceItem, conn=conn) progress.Update() finally: conn.close() progress.PrintStatus()
def convertSourceItems(self, convOptions): """Primary run function to process the contents of the raw source table and convert them into equivalent patient_item, clinical_item, and clinical_item_category entries. Should look for redundancies after the fact to catch repeated conversions. startDate - If provided, only return items whose ordering_date is on or after that date. endDate - If provided, only return items whose ordering_date is before that date. """ log.info("Conversion for items dated %s to %s" % (convOptions.startDate, convOptions.endDate)); progress = ProgressDots(); conn = self.connFactory.connection(); try: # Next round for medications directly from order_med table not addressed in medmix for sourceItem in self.querySourceItems(convOptions, progress=progress, conn=conn): self.convertSourceItem(sourceItem, conn=conn); progress.Update(); finally: conn.close(); progress.PrintStatus();
def syncTable(sourceConn, targetConn, syncTableName, rowIDStrSet=None, formatter=None): if formatter is None: idCol = DBUtil.defaultIDColumn(syncTableName) idQuery = "select %s from %s" % (idCol, syncTableName) # Collect all of the IDs known in the target database and store in memory for rapid lookup print("Querying for IDs from Target Database", file=sys.stderr) targetIdTable = DBUtil.execute(idQuery, conn=targetConn) targetIdSet = set() for row in targetIdTable: targetId = row[0] targetIdSet.add(targetId) # Query data out of the source table, but do it by a cursor so we can stream through large data tables print("Querying for Source Data", file=sys.stderr) dataQuery = "select * from %s" % (syncTableName) sourceCursor = sourceConn.cursor() sourceCursor.execute(dataQuery) colNames = DBUtil.columnNamesFromCursor(sourceCursor) targetCursor = targetConn.cursor() insertQuery = None updateQuery = None progress = ProgressDots() row = sourceCursor.fetchone() while row is not None: dataModel = RowItemModel(row, colNames) if rowIDStrSet is None or str(dataModel[idCol]) in rowIDStrSet: if rowIDStrSet is not None: print("Syncing record: %s" % dataModel[idCol], file=sys.stderr) if dataModel[idCol] not in targetIdSet: # Row does not yet exist in target database, need to insert it if insertQuery is None: insertQuery = DBUtil.buildInsertQuery( syncTableName, list(dataModel.keys())) insertParams = list(dataModel.values()) targetCursor.execute(insertQuery, insertParams) else: # Row already exists in target database, just update values if updateQuery is None: updateQuery = DBUtil.buildUpdateQuery( syncTableName, list(dataModel.keys())) updateParams = [] updateParams.extend(list(dataModel.values())) updateParams.append(dataModel[idCol]) targetCursor.execute(updateQuery, updateParams) if progress.GetCounts() % progress.big == 0: targetConn.commit() row = sourceCursor.fetchone() progress.Update() progress.PrintStatus() targetConn.commit() else: ##Do something with thr formatter ##Set up the formatter theFormatter = formatter(syncTableName, targetConn, includeColumnNames=True, autoCommit=True) #Call DB execute res = DBUtil.execute("select * from %s" % syncTableName, includeColumnNames=True, conn=sourceConn, formatter=theFormatter)