#-----------------------------------------------------------
def __get_request_from_8201(self, request_data):
request = None
try:
pat_ldt = {
'lastnames': request_data['3101'][0],
'firstnames': request_data['3102'][0],
'dob': gmXdtMappings.xdt_8date2iso(request_data['3103'][0])
}
except KeyError, IndexError:
pat_ldt = None
# either get lab request from request id
if '8310' in request_data:
reqid = request_data['8310'][0]
try:
request = gmPathLab.cLabRequest(req_id=reqid, lab=self.__lab_name)
except gmExceptions.ConstructorError:
_log.LogException('cannot get lab request', sys.exc_info(), verbose=0)
# try to verify patient
if request is not None:
if pat_ldt is not None:
pat_db = request.get_patient()
if ((pat_ldt['lastnames'] != pat_db[3]) or
(pat_ldt['firstnames'] != pat_db[2]) or
(pat_ldt['dob'] != pat_db[4].strftime('%Y-%m-%d'))):
_log.Log(gmLog.lErr, 'patient mismatch LDT-Datei <-> Datenbank')
_log.Log(gmLog.lData, 'Datei: %s' % pat_ldt)
_log.Log(gmLog.lData, 'DB: %s' % pat_db)
return None
# or create one from name/dob
class cLDTImporter:
_chunk_starters = ['8000', '8410']
_map_820xline2req_field = {
'8311': 'lab_request_id',
'8301': 'lab_rxd_when',
'8401': 'request_status',
'8405': 'progress_note'
}
def __init__(self, cfg=None):
self._cfg = cfg
# verify db connectivity
# FIXME: port to gmPG2
pool = gmPG.ConnectionPool()
conn = pool.GetConnection('historica')
if conn is None:
_log.Log(gmLog.lErr, 'cannot connect to database')
raise gmExceptions.ConstructorError, 'cannot connect to database'
else:
pool.ReleaseConnection('historica')
#-----------------------------------------------------------
def import_file(self, filename=None):
# verify ldt file
if not os.access(filename, os.R_OK):
_log.Log(gmLog.lErr, 'cannot access LDT file [%s] for reads' % filename)
return False
self.ldt_filename = filename
# verify header of LDT file
if not self.__verify_file_header(self.ldt_filename):
_log.Log(gmLog.lInfo, 'cannot verify file header on [%s]' % self.ldt_filename)
return False
# verify base working directory
self.work_base = self._cfg.get('import', 'work dir base')
if self.work_base is None:
self.work_base = os.path.dirname(self.ldt_filename)
self.work_base = os.path.abspath(os.path.expanduser(self.work_base))
if not os.access(self.work_base, os.W_OK):
_log.Log(gmLog.lErr, 'cannot write to work directory [%s]' % self.work_base)
return False
# create scratch directory
tempfile.tempdir = self.work_base
self.work_dir = tempfile.mktemp()
os.mkdir(self.work_dir, 0700)
# split into parts
source_files = self.__split_file(self.ldt_filename)
if source_files is None:
_log.Log(gmLog.lErr, 'cannot split LDT file [%s]' % self.ldt_filename)
return False
# import requested results
if len(source_files['data']) > 0:
tmp_src_files = copy.copy(source_files['data'])
for request_file in tmp_src_files:
_log.Log(gmLog.lInfo, 'importing request file [%s]' % request_file)
if self.__import_request_file(request_file):
source_files['data'].remove(request_file)
_log.Log(gmLog.lErr, 'success importing request file\n')
else:
_log.Log(gmLog.lErr, 'cannot import request file [%s]\n' % request_file)
else:
_log.Log(gmLog.lData, 'skipping empty LDT file [%s]' % self.ldt_filename)
# cleanup work area
try:
shutil.rmtree(path=self.work_dir, ignore_errors=True)
except Exception:
_log.LogException('cannot cleanup work dir [%s]' % self.work_dir, sys.exc_info(), verbose=0)
# anything left ?
if len(source_files['data']) > 0:
# keep source file for next time
return False
return True
#-----------------------------------------------------------
# internal helpers
#-----------------------------------------------------------
def _verify_8300(self, a_line, field_data):
# FIXME: internal_name is no more
cmd = "select exists(select pk from test_org where internal_name=%s)"
status = gmPG.run_ro_query('historica', cmd, None, field_data)
if status is None:
_log.Log(gmLog.lErr, 'cannot check for lab existance on [%s]' % field_data)
return False
if not status[0][0]:
_log.Log(gmLog.lErr, 'Unbekanntes Labor [%s]' % field_data)
prob = 'Labor unbekannt. Import abgebrochen.' % field_data
sol = 'Labor ergänzen oder vorhandenes Labor anpassen (test_org.internal_name).'
ctxt = 'LDT-Datei [%s], Labor [%s]' % (self.ldt_filename, field_data)
add_todo(problem=prob, solution=sol, context=txt)
return False
self.__lab_name = field_data
return True
#-----------------------------------------------------------
# def _verify_9211(self, a_line, field_data):
# _log.Log(gmLog.lInfo, 'LDT version [%s]' % field_data)
# return True
#-----------------------------------------------------------
_map_field2verifier = {
'8300': _verify_8300
# '9211': _verify_9211
}
#-----------------------------------------------------------
def __verify_file_header(self, filename):
"""Verify that header is suitable for import.
This does not verify whether the header is conforming
to the LDT specs but rather that it is fit for import.
"""
verified_lines = 0
in_header = False
for line in fileinput.input(filename):
tmp = line.replace('\r','')
tmp = tmp.replace('\n','')
line_type = tmp[3:7]
line_data = tmp[7:]
# found start of record
if line_type == '8000':
# header found ?
if line_data == '8220':
in_header = True
continue
# skip prepended junk
if not in_header:
continue
# found record following header
if line_data != '8220':
fileinput.close()
if verified_lines == len(cLDTImporter._map_field2verifier):
return True
_log.Log(gmLog.lErr, 'zuwenige verifizierbare Zeilen im LDT-Datei-Header')
return False
if not in_header:
continue
try:
verify_line = cLDTImporter._map_field2verifier[line_type]
except KeyError:
_log.Log(gmLog.lData, 'kein Handler für Zeilentyp [%s] in LDT-Datei-Header' % line_type)
continue
if verify_line(self, line, line_data):
verified_lines += 1
else:
_log.Log(gmLog.lErr, 'cannot handle LDT file [%s]' % filename)
fileinput.close()
return False
_log.Log(gmLog.lErr, 'LDT file [%s] contains nothing but a header' % filename)
fileinput.close()
return False
#-----------------------------------------------------------
def __split_file(self, filename):
"""Split LDT file.
Splits LDT files into header (record type 8220), data
records (8202, etc) and trailer (8221).
"""
tempfile.tempdir = self.work_dir
source_files = {}
source_files['data'] = []
outname = None
in_header = False
in_trailer = False
for line in fileinput.input(filename):
tmp = line.replace('\r','')
tmp = tmp.replace('\n','')
line_type = tmp[3:7]
line_data = tmp[7:]
if line_type == '8000':
# start headers == start of file
if line_data in ['0020', '8220']:
if not in_header:
header = os.path.join(self.work_dir, 'header.txt')
source_files['header'] = header
outfile = open(header, 'w+b')
in_header = True
# start of trailer
elif line_data in ['8221', '0021']:
in_header = False
# did we have any data records ?
if outname is not None:
# yes, so append them
source_files['data'].append(outname)
if not in_trailer:
outfile.close()
trailer = os.path.join(self.work_dir, 'trailer.txt')
source_files['trailer'] = trailer
outfile = open(trailer, 'w+b')
in_trailer = True
# start of data record
else:
in_header = False
in_trailer = False
outfile.close()
# first record ?
if outname is not None:
# no, so append record name
source_files['data'].append(outname)
outname = os.path.join(self.work_dir, tempfile.mktemp(suffix='.txt'))
outfile = open(outname, 'w+b')
if line_data not in ['8202', '8201']:
_log.Log(gmLog.lWarn, 'unbekannter Satztyp [%s]' % line_data)
# keep line
outfile.write(line)
# end of file
outfile.close()
fileinput.close()
return source_files
#-----------------------------------------------------------
#-----------------------------------------------------------
def __xform_8311(self, request_data):
return request_data['8311'][0].strip()
#-----------------------------------------------------------
def __xform_8301(self, request_data):
return gmXdtMappings.xdt_8date2iso(request_data['8301'][0].strip())
#-----------------------------------------------------------
def __xform_8302(self, request_data):
"""8302: Berichtsdatum"""
if self.__request['results_reported_when'] is None:
self.__request['results_reported_when'] = mxDT.now()
self.__request['results_reported_when'] = mxDT.strptime(
request_data['8302'][0].strip(),
'%d%m%Y',
self.__request['results_reported_when']
)
return None
#-----------------------------------------------------------
def __xform_8303(self, request_data):
"""8303: Berichtszeit"""
if self.__request['results_reported_when'] is None:
self.__request['results_reported_when'] = mxDT.now()
self.__request['results_reported_when'] = mxDT.strptime(
request_data['8303'][0].strip(),
'%H%M',
self.__request['results_reported_when']
)
return None
#-----------------------------------------------------------
def __xform_3103(self, request_data):
tmp = gmXdtMappings.xdt_8date2iso(request_data['3103'][0])
# keep for result records to store
self.__ref_group_str = ' / '.join([self.__ref_group_str, ("geb: %s" % tmp)])
# - sanity check patient dob
cmd = """
select exists(
select 1
from clin.v_pat_items vpi, clin.lab_request lr, dem.identity i
where
vpi.pk_patient=i.pk
and
vpi.pk_item=%s
and
date_trunc('day', i.dob)=%s::timestamp
)"""
data = gmPG.run_ro_query('personalia', cmd, None, self.__request['pk_item'], tmp)
if data is None:
_log.Log(gmLog.lErr, 'cannot sanity check patient dob [%s]' % tmp)
if not data[0]:
_log.Log(gmLog.lErr, 'lab used [%s] as reference dob while patient has different dob !' % tmp)
self.__ref_group_str = "!!!*** Im Labor wurde vermutlich ein falsches Referenzalter verwendet. ***!!!\n%s" % self.__ref_group_str
return None
#-----------------------------------------------------------
def __xform_8401(self, request_data):
try:
req_stat = gmXdtMappings.map_Befundstatus_xdt2gm[request_data['8401'][0].strip()]
except KeyError:
_log.Log(gmLog.lErr, 'unbekannter Befundstatus [%s] (Feld 8401, Regel 135)' % request_data['8401'][0])
req_stat = 'preliminary'
# sanity check
if (not self.__request['is_pending']) and (req_stat != 'final'):
prob = 'kein Befund für [%s] mehr erwartet, aber Befund mit Status [%s] erhalten)' % (self.__request['request_id'], req_stat)
sol = 'Befund wird trotzdem importiert. Bitte Befunde auf Duplikate überprüfen.'
ctxt = 'Patient: %s, Labor [%s], LDT-Datei [%s], Probe [%s], (Feld 8401, Regel 135)' % (self.__request.get_patient(), self.__lab_name, self.ldt_filename, self.__request['request_id'])
add_todo(problem=prob, solution=sol, context=ctxt)
_log.Log(gmLog.lWarn, prob)
_log.Log(gmLog.lWarn, ctxt)
# don't force is_pending back to true just because
# a wrong status was received from the lab
req_stat = 'final'
if req_stat == 'final':
self.__request['is_pending'] = 'false'
else:
self.__request['is_pending'] = 'true'
return req_stat
#-----------------------------------------------------------
def __xform_8405(self, request_data):
new_comment = '\n'.join(request_data['8405'])
old_narrative = self.__request['progress_note']
if old_narrative is None:
return new_comment
if old_narrative.find(new_comment) == -1:
old_narrative = old_narrative + '\n' + new_comment
return old_narrative
#-----------------------------------------------------------
def __xform_8407(self, request_data):
tmp = request_data['8407'][0]
# keep this so test results can store it
self.__ref_group_str = ' / '.join([gmXdtMappings.map_8407_2str[tmp], self.__ref_group_str])
# - sanity check patient gender/age
# I discussed the age cutoff with a pediatrician and
# we came to the conclusion that a useful value for
# child/adult age separation in terms of lab results
# would be 12 years.
# - get patient gender/age
cmd = """
select
i.gender,
case when i.dob < (now() - '12 years'::interval)
then false
else true
end as is_child
from v_pat_items vpi, lab_request lr, identity i
where
vpi.id_patient=i.pk
and
vpi.pk_item=%s"""
data = gmPG.run_ro_query('personalia', cmd, None, self.__request['pk_item'])
if data is None:
_log.Log(gmLog.lErr, 'cannot sanity check patient ref group [%s]' % gmXdtMappings.map_8407_2str[tmp])
elif len(data) == 0:
_log.Log(gmLog.lErr, 'cannot sanity check patient ref group [%s]' % gmXdtMappings.map_8407_2str[tmp])
else:
gender = data[0]
is_child = data[1]
if tmp in [0,6]:
self.__ref_group_str = "%s\n* Kann Referenzgruppe nicht mit Patient abgleichen. *" % self.__ref_group_str
else:
# Kind
if (((tmp in [3,4,5]) and (not is_child)) or
# männlich
((tmp in [1,4]) and (gender != 'm')) or
# weiblich
((tmp in [2,5]) and (gender != 'f'))):
_log.Log(gmLog.lErr, 'lab thinks patient is [%s] but patient is [%s:child->%s]' % (gmXdtMappings.map_8407_2str[tmp], gender, is_child))
self.__ref_group_str = "!!!*** Im Labor wurde vermutlich eine falsche Referenzgruppe verwendet (Alter/Geschlecht). ***!!!\n%s" % self.__ref_group_str
return None
#-----------------------------------------------------------
__820xline_handler = {
'0020': None,
'9105': None,
'8000': None,
'8100': None,
'8310': None,
'8311': __xform_8311,
'8301': __xform_8301,
'8302': __xform_8302,
'8303': __xform_8303,
'3100': None,
'3101': None,
'3102': None,
'3103': __xform_3103,
'3104': None,
'8401': __xform_8401,
'8403': None, # Gebührenordnung
'8405': __xform_8405,
'8407': __xform_8407,
'8609': None # Gebührenordnung
}
#-----------------------------------------------------------
def __handle_8202(self, request_data):
# essential fields
try:
reqid = request_data['8310'][0]
except KeyError, IndexError:
# FIXME: todo item
_log.Log(gmLog.lErr, 'Satz vom Typ [8000:%s] enthält keine Probennummer' % request_data['8000'][0])
return False
# get lab request
try:
self.__request = gmPathLab.cLabRequest(req_id=reqid, lab=self.__lab_name)
except gmExceptions.ConstructorError:
prob = 'Kann keine Patientenzuordnung der Probe finden.'
sol = 'Zuordnung der Probe zu einem Patienten prüfen. Falls doch vorhanden, Systembetreuer verständigen.'
ctxt = 'Labor [%s], Probe [%s], LDT-Datei [%s]' % (self.__lab_name, reqid, self.ldt_filename)
add_todo(problem=prob, solution=sol, context=ctxt)
_log.LogException('cannot get lab request', sys.exc_info(), verbose=0)
return False
# update fields in request from request_data
for line_type in request_data.keys():
# get handler
try:
handle_line = cLDTImporter.__820xline_handler[line_type]
except KeyError:
_log.LogException('no handler for line [%s:%s] in [8000:8201] record' % (line_type, request_data[line_type]), sys.exc_info(), verbose=0)
continue
# ignore line
if handle_line is None:
try:
name = gmXdtMappings.xdt_id_map[line_type]
except KeyError:
name = '?'
_log.Log(gmLog.lData, 'skipping [%s] (%s)' % (line_type, name))
continue
# handle line
line_data = handle_line(self, request_data)
if line_data is False:
# FIXME: todo item
_log.Log(gmLog.lErr, 'handling line [%s] failed' % line_type)
return False
try:
self.__request[cLDTImporter._map_820xline2req_field[line_type]] = line_data
except KeyError:
pass
self.__request.save_payload()
return True
#-----------------------------------------------------------
def __get_request_from_8201(self, request_data):
request = None
try:
pat_ldt = {
'lastnames': request_data['3101'][0],
'firstnames': request_data['3102'][0],
'dob': gmXdtMappings.xdt_8date2iso(request_data['3103'][0])
}
except KeyError, IndexError:
pat_ldt = None
# either get lab request from request id
if request_data.has_key('8310'):
reqid = request_data['8310'][0]
try:
request = gmPathLab.cLabRequest(req_id=reqid, lab=self.__lab_name)
except gmExceptions.ConstructorError:
_log.LogException('cannot get lab request', sys.exc_info(), verbose=0)
# try to verify patient
if request is not None:
if pat_ldt is not None:
pat_db = request.get_patient()
if ((pat_ldt['lastnames'] != pat_db[3]) or
(pat_ldt['firstnames'] != pat_db[2]) or
(pat_ldt['dob'] != pat_db[4].strftime('%Y-%m-%d'))):
_log.Log(gmLog.lErr, 'patient mismatch LDT-Datei <-> Datenbank')
_log.Log(gmLog.lData, 'Datei: %s' % pat_ldt)
_log.Log(gmLog.lData, 'DB: %s' % pat_db)
return None
# or create one from name/dob
