table_out = table_dict_name + '_import'
if myspot.sql_tab_exists(target_db, table_out):
print "WARNING: table %s already exists in database %s" % (target_db, table_out)
if args.replace_table:
stmt = "DROP TABLE IF EXISTS %s " % (table_out)
else:
print "ERROR: You must specify the '-replace' option to delete existing tables"
sys.exit(1)
# check you can open the dictionares OK
tdict = myspot.get_yaml_dict('table', local_dict=True)
fdict = myspot.get_yaml_dict('field', local_dict=True)
# falias_dict = myspot.reverse_field_dict(local_dict = True)
# now focus the falias_dict on the likely tables
falias_dict = myspot.make_talias_dict(fdict)
talias_dict = falias_dict['no_alias']
# print len(talias_dict)
if table_dict_name in falias_dict:
# print table_dict_name
# print falias_dict[table_dict_name]
talias_dict.update(falias_dict[table_dict_name])
talias_dict = {i[0].lower():i[1] for i in talias_dict.items()}
debug = False
if debug:
print len(talias_dict)
print talias_dict
sys.exit(1)
# first check there are no identical field names
ttemp =[f['fname'] for f in fdict]
# print len(row_hashes)
# print len(set(row_hashes))
rows_clean = []
index_errors = []
hash_rows = zip(row_hashes, rows)
# =====================================================================================
# = Update rows with corrections from dvr where those corrections affect primary keys =
# =====================================================================================
from myspot import make_talias_dict
from myspot import clean_field_names
from myspot import sql_2mysql
fdict_lookup = {}
talias_dict = make_talias_dict(fdict)
falias_dict = talias_dict['no_alias']
if t_tab in talias_dict:
falias_dict.update(talias_dict[t_tab])
for f in fdict:
fdict_lookup[f['fname']] = f
corrections = False
if 'corrections_table' in tspec and tspec['corrections_table'] is not None:
corrections = True
corrections_table = tspec['corrections_table']
corr_fields = pkeys[:] # shallow copy pkeys
corr_fields.extend([
"field", "value", "validation_message", "new_response_code",
"new_response_value", 'new_response_note'
])
# print len(set(row_hashes))
rows_clean = []
index_errors = []
hash_rows = zip(row_hashes, rows)
# =====================================================================================
# = Update rows with corrections from dvr where those corrections affect primary keys =
# =====================================================================================
from myspot import make_talias_dict
from myspot import clean_field_names
from myspot import sql_2mysql
fdict_lookup = {}
talias_dict = make_talias_dict(fdict)
falias_dict = talias_dict['no_alias']
if t_tab in talias_dict:
falias_dict.update(talias_dict[t_tab])
for f in fdict:
fdict_lookup[f['fname']] = f
corrections = False
if 'corrections_table' in tspec and tspec['corrections_table'] is not None:
corrections = True
corrections_table = tspec['corrections_table']
corr_fields = pkeys[:] # shallow copy pkeys
corr_fields.extend(
["field", "value", "validation_message",
"new_response_code", "new_response_value", 'new_response_note'])
# select all entries with a 2 (delete item) or 3 (update item) response
table_out = table_dict_name + '_import'
if myspot.sql_tab_exists(target_db, table_out):
print "WARNING: table %s already exists in database %s" % (target_db,
table_out)
if args.replace_table:
stmt = "DROP TABLE IF EXISTS %s " % (table_out)
else:
print "ERROR: You must specify the '-replace' option to delete existing tables"
sys.exit(1)
# check you can open the dictionares OK
tdict = myspot.get_yaml_dict('table', local_dict=True)
fdict = myspot.get_yaml_dict('field', local_dict=True)
# falias_dict = myspot.reverse_field_dict(local_dict = True)
# now focus the falias_dict on the likely tables
falias_dict = myspot.make_talias_dict(fdict)
talias_dict = falias_dict['no_alias']
# print len(talias_dict)
if table_dict_name in falias_dict:
# print table_dict_name
# print falias_dict[table_dict_name]
talias_dict.update(falias_dict[table_dict_name])
talias_dict = {i[0].lower(): i[1] for i in talias_dict.items()}
debug = False
if debug:
print len(talias_dict)
print talias_dict
sys.exit(1)
# first check there are no identical field names
ttemp = [f['fname'] for f in fdict]
if sitecode == 'all':
sitecodes.extend(['ucl', 'ket', 'sou', 'med', 'pol', 'yeo', 'nor', 'lis', 'har', 'rvi', 'fre'])
else:
sitecodes.append(sitecode)
# Pull fdict lookup now because need to look-up hosp name
fdict_lookup = get_yaml_dict('field', return_type='dictionary', local_dict=True)
# open the pickled data - this will take a long time!
pickle_rows = open_pickles()
# =======================
# = Pull table spec etc =
# =======================
tdict_lookup = get_yaml_dict('table', return_type='dictionary', local_dict=True)
talias_dict = make_talias_dict(get_yaml_dict('field', local_dict=True))
source_tables = {
'idpid': '(SPOT)id web portal',
'idpatient': 'Initial assessment',
'idvisit': 'Daily assessment',
'idlab': 'Laboratory flow chart',
'idlabuclh': 'Biological sampling (UCLH only)'
}
# ===================================
# = Define checks and DVR variables =
# ===================================
# Check dictionary
vdict = get_yaml_dict('checks', local_dict=True)
# build a dictionary of check names and messages
# Pull fdict lookup now because need to look-up hosp name
fdict_lookup = get_yaml_dict('field',
return_type='dictionary',
local_dict=True)
# open the pickled data - this will take a long time!
pickle_rows = open_pickles()
# =======================
# = Pull table spec etc =
# =======================
tdict_lookup = get_yaml_dict('table',
return_type='dictionary',
local_dict=True)
talias_dict = make_talias_dict(get_yaml_dict('field', local_dict=True))
source_tables = {
'idpid': '(SPOT)id web portal',
'idpatient': 'Initial assessment',
'idvisit': 'Daily assessment',
'idlab': 'Laboratory flow chart',
'idlabuclh': 'Biological sampling (UCLH only)'
}
# ===================================
# = Define checks and DVR variables =
# ===================================
# Check dictionary
vdict = get_yaml_dict('checks', local_dict=True)
# build a dictionary of check names and messages