table_out = table_dict_name + '_import'
if myspot.sql_tab_exists(target_db, table_out):
    print "WARNING: table %s already exists in database %s" % (target_db, table_out)
    if args.replace_table:
        stmt = "DROP TABLE IF EXISTS %s " % (table_out)
    else:
        print "ERROR: You must specify the '-replace' option to delete existing tables"
        sys.exit(1)

# check you can open the dictionares OK
tdict = myspot.get_yaml_dict('table', local_dict=True)
fdict = myspot.get_yaml_dict('field', local_dict=True)
# falias_dict = myspot.reverse_field_dict(local_dict = True)
# now focus the falias_dict on the likely tables
falias_dict = myspot.make_talias_dict(fdict)
talias_dict = falias_dict['no_alias']
# print len(talias_dict)
if table_dict_name in falias_dict:
    # print table_dict_name
    # print falias_dict[table_dict_name]
    talias_dict.update(falias_dict[table_dict_name])
talias_dict = {i[0].lower():i[1] for i in talias_dict.items()}
debug = False
if debug:
    print len(talias_dict)
    print talias_dict
    sys.exit(1)

# first check there are no identical field names
ttemp =[f['fname'] for f in fdict]
Example #2
0
# print len(row_hashes)
# print len(set(row_hashes))

rows_clean = []
index_errors = []
hash_rows = zip(row_hashes, rows)

#  =====================================================================================
#  = Update rows with corrections from dvr where those corrections affect primary keys =
#  =====================================================================================
from myspot import make_talias_dict
from myspot import clean_field_names
from myspot import sql_2mysql

fdict_lookup = {}
talias_dict = make_talias_dict(fdict)
falias_dict = talias_dict['no_alias']
if t_tab in talias_dict:
    falias_dict.update(talias_dict[t_tab])

for f in fdict:
    fdict_lookup[f['fname']] = f
corrections = False
if 'corrections_table' in tspec and tspec['corrections_table'] is not None:
    corrections = True
    corrections_table = tspec['corrections_table']
    corr_fields = pkeys[:]  # shallow copy pkeys
    corr_fields.extend([
        "field", "value", "validation_message", "new_response_code",
        "new_response_value", 'new_response_note'
    ])
Example #3
0
# print len(set(row_hashes))


rows_clean = []
index_errors = []
hash_rows = zip(row_hashes, rows)

#  =====================================================================================
#  = Update rows with corrections from dvr where those corrections affect primary keys =
#  =====================================================================================
from myspot import make_talias_dict
from myspot import clean_field_names
from myspot import sql_2mysql

fdict_lookup = {}
talias_dict = make_talias_dict(fdict)
falias_dict = talias_dict['no_alias']
if t_tab in talias_dict:
    falias_dict.update(talias_dict[t_tab])

for f in fdict:
    fdict_lookup[f['fname']] = f
corrections = False
if 'corrections_table' in tspec and tspec['corrections_table'] is not None:
    corrections = True
    corrections_table = tspec['corrections_table']
    corr_fields = pkeys[:]  # shallow copy pkeys
    corr_fields.extend(
            ["field", "value", "validation_message",
            "new_response_code", "new_response_value", 'new_response_note'])
    # select all entries with a 2 (delete item) or 3 (update item) response
Example #4
0
table_out = table_dict_name + '_import'
if myspot.sql_tab_exists(target_db, table_out):
    print "WARNING: table %s already exists in database %s" % (target_db,
                                                               table_out)
    if args.replace_table:
        stmt = "DROP TABLE IF EXISTS %s " % (table_out)
    else:
        print "ERROR: You must specify the '-replace' option to delete existing tables"
        sys.exit(1)

# check you can open the dictionares OK
tdict = myspot.get_yaml_dict('table', local_dict=True)
fdict = myspot.get_yaml_dict('field', local_dict=True)
# falias_dict = myspot.reverse_field_dict(local_dict = True)
# now focus the falias_dict on the likely tables
falias_dict = myspot.make_talias_dict(fdict)
talias_dict = falias_dict['no_alias']
# print len(talias_dict)
if table_dict_name in falias_dict:
    # print table_dict_name
    # print falias_dict[table_dict_name]
    talias_dict.update(falias_dict[table_dict_name])
talias_dict = {i[0].lower(): i[1] for i in talias_dict.items()}
debug = False
if debug:
    print len(talias_dict)
    print talias_dict
    sys.exit(1)

# first check there are no identical field names
ttemp = [f['fname'] for f in fdict]
Example #5
0
    if sitecode == 'all':
        sitecodes.extend(['ucl', 'ket', 'sou', 'med', 'pol', 'yeo', 'nor', 'lis', 'har', 'rvi', 'fre'])
    else:
        sitecodes.append(sitecode)

# Pull fdict lookup now because need to look-up hosp name

fdict_lookup = get_yaml_dict('field', return_type='dictionary', local_dict=True)
# open the pickled data - this will take a long time!
pickle_rows = open_pickles()

#  =======================
#  = Pull table spec etc =
#  =======================
tdict_lookup = get_yaml_dict('table', return_type='dictionary', local_dict=True)
talias_dict = make_talias_dict(get_yaml_dict('field', local_dict=True))
source_tables = {
    'idpid': '(SPOT)id web portal',
    'idpatient': 'Initial assessment',
    'idvisit': 'Daily assessment',
    'idlab': 'Laboratory flow chart',
    'idlabuclh': 'Biological sampling (UCLH only)'
}

#  ===================================
#  = Define checks and DVR variables =
#  ===================================
# Check dictionary
vdict = get_yaml_dict('checks', local_dict=True)

# build a dictionary of check names and messages
Example #6
0
# Pull fdict lookup now because need to look-up hosp name

fdict_lookup = get_yaml_dict('field',
                             return_type='dictionary',
                             local_dict=True)
# open the pickled data - this will take a long time!
pickle_rows = open_pickles()

#  =======================
#  = Pull table spec etc =
#  =======================
tdict_lookup = get_yaml_dict('table',
                             return_type='dictionary',
                             local_dict=True)
talias_dict = make_talias_dict(get_yaml_dict('field', local_dict=True))
source_tables = {
    'idpid': '(SPOT)id web portal',
    'idpatient': 'Initial assessment',
    'idvisit': 'Daily assessment',
    'idlab': 'Laboratory flow chart',
    'idlabuclh': 'Biological sampling (UCLH only)'
}

#  ===================================
#  = Define checks and DVR variables =
#  ===================================
# Check dictionary
vdict = get_yaml_dict('checks', local_dict=True)

# build a dictionary of check names and messages