#----------------------------------------------------------- def __get_request_from_8201(self, request_data): request = None try: pat_ldt = { 'lastnames': request_data['3101'][0], 'firstnames': request_data['3102'][0], 'dob': gmXdtMappings.xdt_8date2iso(request_data['3103'][0]) } except KeyError, IndexError: pat_ldt = None # either get lab request from request id if '8310' in request_data: reqid = request_data['8310'][0] try: request = gmPathLab.cLabRequest(req_id=reqid, lab=self.__lab_name) except gmExceptions.ConstructorError: _log.LogException('cannot get lab request', sys.exc_info(), verbose=0) # try to verify patient if request is not None: if pat_ldt is not None: pat_db = request.get_patient() if ((pat_ldt['lastnames'] != pat_db[3]) or (pat_ldt['firstnames'] != pat_db[2]) or (pat_ldt['dob'] != pat_db[4].strftime('%Y-%m-%d'))): _log.Log(gmLog.lErr, 'patient mismatch LDT-Datei <-> Datenbank') _log.Log(gmLog.lData, 'Datei: %s' % pat_ldt) _log.Log(gmLog.lData, 'DB: %s' % pat_db) return None # or create one from name/dob
class cLDTImporter: _chunk_starters = ['8000', '8410'] _map_820xline2req_field = { '8311': 'lab_request_id', '8301': 'lab_rxd_when', '8401': 'request_status', '8405': 'progress_note' } def __init__(self, cfg=None): self._cfg = cfg # verify db connectivity # FIXME: port to gmPG2 pool = gmPG.ConnectionPool() conn = pool.GetConnection('historica') if conn is None: _log.Log(gmLog.lErr, 'cannot connect to database') raise gmExceptions.ConstructorError, 'cannot connect to database' else: pool.ReleaseConnection('historica') #----------------------------------------------------------- def import_file(self, filename=None): # verify ldt file if not os.access(filename, os.R_OK): _log.Log(gmLog.lErr, 'cannot access LDT file [%s] for reads' % filename) return False self.ldt_filename = filename # verify header of LDT file if not self.__verify_file_header(self.ldt_filename): _log.Log(gmLog.lInfo, 'cannot verify file header on [%s]' % self.ldt_filename) return False # verify base working directory self.work_base = self._cfg.get('import', 'work dir base') if self.work_base is None: self.work_base = os.path.dirname(self.ldt_filename) self.work_base = os.path.abspath(os.path.expanduser(self.work_base)) if not os.access(self.work_base, os.W_OK): _log.Log(gmLog.lErr, 'cannot write to work directory [%s]' % self.work_base) return False # create scratch directory tempfile.tempdir = self.work_base self.work_dir = tempfile.mktemp() os.mkdir(self.work_dir, 0700) # split into parts source_files = self.__split_file(self.ldt_filename) if source_files is None: _log.Log(gmLog.lErr, 'cannot split LDT file [%s]' % self.ldt_filename) return False # import requested results if len(source_files['data']) > 0: tmp_src_files = copy.copy(source_files['data']) for request_file in tmp_src_files: _log.Log(gmLog.lInfo, 'importing request file [%s]' % request_file) if self.__import_request_file(request_file): source_files['data'].remove(request_file) _log.Log(gmLog.lErr, 'success importing request file\n') else: _log.Log(gmLog.lErr, 'cannot import request file [%s]\n' % request_file) else: _log.Log(gmLog.lData, 'skipping empty LDT file [%s]' % self.ldt_filename) # cleanup work area try: shutil.rmtree(path=self.work_dir, ignore_errors=True) except Exception: _log.LogException('cannot cleanup work dir [%s]' % self.work_dir, sys.exc_info(), verbose=0) # anything left ? if len(source_files['data']) > 0: # keep source file for next time return False return True #----------------------------------------------------------- # internal helpers #----------------------------------------------------------- def _verify_8300(self, a_line, field_data): # FIXME: internal_name is no more cmd = "select exists(select pk from test_org where internal_name=%s)" status = gmPG.run_ro_query('historica', cmd, None, field_data) if status is None: _log.Log(gmLog.lErr, 'cannot check for lab existance on [%s]' % field_data) return False if not status[0][0]: _log.Log(gmLog.lErr, 'Unbekanntes Labor [%s]' % field_data) prob = 'Labor unbekannt. Import abgebrochen.' % field_data sol = 'Labor ergänzen oder vorhandenes Labor anpassen (test_org.internal_name).' ctxt = 'LDT-Datei [%s], Labor [%s]' % (self.ldt_filename, field_data) add_todo(problem=prob, solution=sol, context=txt) return False self.__lab_name = field_data return True #----------------------------------------------------------- # def _verify_9211(self, a_line, field_data): # _log.Log(gmLog.lInfo, 'LDT version [%s]' % field_data) # return True #----------------------------------------------------------- _map_field2verifier = { '8300': _verify_8300 # '9211': _verify_9211 } #----------------------------------------------------------- def __verify_file_header(self, filename): """Verify that header is suitable for import. This does not verify whether the header is conforming to the LDT specs but rather that it is fit for import. """ verified_lines = 0 in_header = False for line in fileinput.input(filename): tmp = line.replace('\r','') tmp = tmp.replace('\n','') line_type = tmp[3:7] line_data = tmp[7:] # found start of record if line_type == '8000': # header found ? if line_data == '8220': in_header = True continue # skip prepended junk if not in_header: continue # found record following header if line_data != '8220': fileinput.close() if verified_lines == len(cLDTImporter._map_field2verifier): return True _log.Log(gmLog.lErr, 'zuwenige verifizierbare Zeilen im LDT-Datei-Header') return False if not in_header: continue try: verify_line = cLDTImporter._map_field2verifier[line_type] except KeyError: _log.Log(gmLog.lData, 'kein Handler für Zeilentyp [%s] in LDT-Datei-Header' % line_type) continue if verify_line(self, line, line_data): verified_lines += 1 else: _log.Log(gmLog.lErr, 'cannot handle LDT file [%s]' % filename) fileinput.close() return False _log.Log(gmLog.lErr, 'LDT file [%s] contains nothing but a header' % filename) fileinput.close() return False #----------------------------------------------------------- def __split_file(self, filename): """Split LDT file. Splits LDT files into header (record type 8220), data records (8202, etc) and trailer (8221). """ tempfile.tempdir = self.work_dir source_files = {} source_files['data'] = [] outname = None in_header = False in_trailer = False for line in fileinput.input(filename): tmp = line.replace('\r','') tmp = tmp.replace('\n','') line_type = tmp[3:7] line_data = tmp[7:] if line_type == '8000': # start headers == start of file if line_data in ['0020', '8220']: if not in_header: header = os.path.join(self.work_dir, 'header.txt') source_files['header'] = header outfile = open(header, 'w+b') in_header = True # start of trailer elif line_data in ['8221', '0021']: in_header = False # did we have any data records ? if outname is not None: # yes, so append them source_files['data'].append(outname) if not in_trailer: outfile.close() trailer = os.path.join(self.work_dir, 'trailer.txt') source_files['trailer'] = trailer outfile = open(trailer, 'w+b') in_trailer = True # start of data record else: in_header = False in_trailer = False outfile.close() # first record ? if outname is not None: # no, so append record name source_files['data'].append(outname) outname = os.path.join(self.work_dir, tempfile.mktemp(suffix='.txt')) outfile = open(outname, 'w+b') if line_data not in ['8202', '8201']: _log.Log(gmLog.lWarn, 'unbekannter Satztyp [%s]' % line_data) # keep line outfile.write(line) # end of file outfile.close() fileinput.close() return source_files #----------------------------------------------------------- #----------------------------------------------------------- def __xform_8311(self, request_data): return request_data['8311'][0].strip() #----------------------------------------------------------- def __xform_8301(self, request_data): return gmXdtMappings.xdt_8date2iso(request_data['8301'][0].strip()) #----------------------------------------------------------- def __xform_8302(self, request_data): """8302: Berichtsdatum""" if self.__request['results_reported_when'] is None: self.__request['results_reported_when'] = mxDT.now() self.__request['results_reported_when'] = mxDT.strptime( request_data['8302'][0].strip(), '%d%m%Y', self.__request['results_reported_when'] ) return None #----------------------------------------------------------- def __xform_8303(self, request_data): """8303: Berichtszeit""" if self.__request['results_reported_when'] is None: self.__request['results_reported_when'] = mxDT.now() self.__request['results_reported_when'] = mxDT.strptime( request_data['8303'][0].strip(), '%H%M', self.__request['results_reported_when'] ) return None #----------------------------------------------------------- def __xform_3103(self, request_data): tmp = gmXdtMappings.xdt_8date2iso(request_data['3103'][0]) # keep for result records to store self.__ref_group_str = ' / '.join([self.__ref_group_str, ("geb: %s" % tmp)]) # - sanity check patient dob cmd = """ select exists( select 1 from clin.v_pat_items vpi, clin.lab_request lr, dem.identity i where vpi.pk_patient=i.pk and vpi.pk_item=%s and date_trunc('day', i.dob)=%s::timestamp )""" data = gmPG.run_ro_query('personalia', cmd, None, self.__request['pk_item'], tmp) if data is None: _log.Log(gmLog.lErr, 'cannot sanity check patient dob [%s]' % tmp) if not data[0]: _log.Log(gmLog.lErr, 'lab used [%s] as reference dob while patient has different dob !' % tmp) self.__ref_group_str = "!!!*** Im Labor wurde vermutlich ein falsches Referenzalter verwendet. ***!!!\n%s" % self.__ref_group_str return None #----------------------------------------------------------- def __xform_8401(self, request_data): try: req_stat = gmXdtMappings.map_Befundstatus_xdt2gm[request_data['8401'][0].strip()] except KeyError: _log.Log(gmLog.lErr, 'unbekannter Befundstatus [%s] (Feld 8401, Regel 135)' % request_data['8401'][0]) req_stat = 'preliminary' # sanity check if (not self.__request['is_pending']) and (req_stat != 'final'): prob = 'kein Befund für [%s] mehr erwartet, aber Befund mit Status [%s] erhalten)' % (self.__request['request_id'], req_stat) sol = 'Befund wird trotzdem importiert. Bitte Befunde auf Duplikate überprüfen.' ctxt = 'Patient: %s, Labor [%s], LDT-Datei [%s], Probe [%s], (Feld 8401, Regel 135)' % (self.__request.get_patient(), self.__lab_name, self.ldt_filename, self.__request['request_id']) add_todo(problem=prob, solution=sol, context=ctxt) _log.Log(gmLog.lWarn, prob) _log.Log(gmLog.lWarn, ctxt) # don't force is_pending back to true just because # a wrong status was received from the lab req_stat = 'final' if req_stat == 'final': self.__request['is_pending'] = 'false' else: self.__request['is_pending'] = 'true' return req_stat #----------------------------------------------------------- def __xform_8405(self, request_data): new_comment = '\n'.join(request_data['8405']) old_narrative = self.__request['progress_note'] if old_narrative is None: return new_comment if old_narrative.find(new_comment) == -1: old_narrative = old_narrative + '\n' + new_comment return old_narrative #----------------------------------------------------------- def __xform_8407(self, request_data): tmp = request_data['8407'][0] # keep this so test results can store it self.__ref_group_str = ' / '.join([gmXdtMappings.map_8407_2str[tmp], self.__ref_group_str]) # - sanity check patient gender/age # I discussed the age cutoff with a pediatrician and # we came to the conclusion that a useful value for # child/adult age separation in terms of lab results # would be 12 years. # - get patient gender/age cmd = """ select i.gender, case when i.dob < (now() - '12 years'::interval) then false else true end as is_child from v_pat_items vpi, lab_request lr, identity i where vpi.id_patient=i.pk and vpi.pk_item=%s""" data = gmPG.run_ro_query('personalia', cmd, None, self.__request['pk_item']) if data is None: _log.Log(gmLog.lErr, 'cannot sanity check patient ref group [%s]' % gmXdtMappings.map_8407_2str[tmp]) elif len(data) == 0: _log.Log(gmLog.lErr, 'cannot sanity check patient ref group [%s]' % gmXdtMappings.map_8407_2str[tmp]) else: gender = data[0] is_child = data[1] if tmp in [0,6]: self.__ref_group_str = "%s\n* Kann Referenzgruppe nicht mit Patient abgleichen. *" % self.__ref_group_str else: # Kind if (((tmp in [3,4,5]) and (not is_child)) or # männlich ((tmp in [1,4]) and (gender != 'm')) or # weiblich ((tmp in [2,5]) and (gender != 'f'))): _log.Log(gmLog.lErr, 'lab thinks patient is [%s] but patient is [%s:child->%s]' % (gmXdtMappings.map_8407_2str[tmp], gender, is_child)) self.__ref_group_str = "!!!*** Im Labor wurde vermutlich eine falsche Referenzgruppe verwendet (Alter/Geschlecht). ***!!!\n%s" % self.__ref_group_str return None #----------------------------------------------------------- __820xline_handler = { '0020': None, '9105': None, '8000': None, '8100': None, '8310': None, '8311': __xform_8311, '8301': __xform_8301, '8302': __xform_8302, '8303': __xform_8303, '3100': None, '3101': None, '3102': None, '3103': __xform_3103, '3104': None, '8401': __xform_8401, '8403': None, # Gebührenordnung '8405': __xform_8405, '8407': __xform_8407, '8609': None # Gebührenordnung } #----------------------------------------------------------- def __handle_8202(self, request_data): # essential fields try: reqid = request_data['8310'][0] except KeyError, IndexError: # FIXME: todo item _log.Log(gmLog.lErr, 'Satz vom Typ [8000:%s] enthält keine Probennummer' % request_data['8000'][0]) return False # get lab request try: self.__request = gmPathLab.cLabRequest(req_id=reqid, lab=self.__lab_name) except gmExceptions.ConstructorError: prob = 'Kann keine Patientenzuordnung der Probe finden.' sol = 'Zuordnung der Probe zu einem Patienten prüfen. Falls doch vorhanden, Systembetreuer verständigen.' ctxt = 'Labor [%s], Probe [%s], LDT-Datei [%s]' % (self.__lab_name, reqid, self.ldt_filename) add_todo(problem=prob, solution=sol, context=ctxt) _log.LogException('cannot get lab request', sys.exc_info(), verbose=0) return False # update fields in request from request_data for line_type in request_data.keys(): # get handler try: handle_line = cLDTImporter.__820xline_handler[line_type] except KeyError: _log.LogException('no handler for line [%s:%s] in [8000:8201] record' % (line_type, request_data[line_type]), sys.exc_info(), verbose=0) continue # ignore line if handle_line is None: try: name = gmXdtMappings.xdt_id_map[line_type] except KeyError: name = '?' _log.Log(gmLog.lData, 'skipping [%s] (%s)' % (line_type, name)) continue # handle line line_data = handle_line(self, request_data) if line_data is False: # FIXME: todo item _log.Log(gmLog.lErr, 'handling line [%s] failed' % line_type) return False try: self.__request[cLDTImporter._map_820xline2req_field[line_type]] = line_data except KeyError: pass self.__request.save_payload() return True
#----------------------------------------------------------- def __get_request_from_8201(self, request_data): request = None try: pat_ldt = { 'lastnames': request_data['3101'][0], 'firstnames': request_data['3102'][0], 'dob': gmXdtMappings.xdt_8date2iso(request_data['3103'][0]) } except KeyError, IndexError: pat_ldt = None # either get lab request from request id if request_data.has_key('8310'): reqid = request_data['8310'][0] try: request = gmPathLab.cLabRequest(req_id=reqid, lab=self.__lab_name) except gmExceptions.ConstructorError: _log.LogException('cannot get lab request', sys.exc_info(), verbose=0) # try to verify patient if request is not None: if pat_ldt is not None: pat_db = request.get_patient() if ((pat_ldt['lastnames'] != pat_db[3]) or (pat_ldt['firstnames'] != pat_db[2]) or (pat_ldt['dob'] != pat_db[4].strftime('%Y-%m-%d'))): _log.Log(gmLog.lErr, 'patient mismatch LDT-Datei <-> Datenbank') _log.Log(gmLog.lData, 'Datei: %s' % pat_ldt) _log.Log(gmLog.lData, 'DB: %s' % pat_db) return None # or create one from name/dob