def display_export_forms(exportid, detid=0, notes=False):
if (not (cfg.is_a_dfc_dataframe_loaded())):
cfg.drop_config_value(cfg.CURRENT_EXPORT_DF)
if (exportid == dem.EXPORT_TB_ONLY):
clear_data_export_data()
dew.display_dc_export_forms(exportid, detid, notes)
def display_sequence_save_subset(df_title, df, auto=False):
"""
* --------------------------------------------------------------------------
* function : display current df subset run
*
* parms :
* df - dataframe to subset from
*
* returns : N/A
* --------------------------------------------------------
"""
df_stats_table = get_df_stats_table(df_title, df)
df_cols_table = get_column_stats_table(df_title, df)
from dfcleanser.common.html_widgets import InputForm
subset_input_form = InputForm(
get_saved_save_input_id, get_saved_save_input_idList,
get_saved_save_input_labelList, get_saved_save_input_typeList,
get_saved_save_input_placeholderList, get_saved_save_input_jsList,
get_saved_save_input_reqList)
selectDicts = []
dropsel = {"default": "False", "list": ["True", "False"]}
selectDicts.append(dropsel)
get_select_defaults(subset_input_form, get_saved_save_input_id,
get_saved_save_input_idList,
get_saved_save_input_typeList, selectDicts)
subset_input_form.set_shortForm(False)
subset_input_form.set_gridwidth(680)
subset_input_form.set_custombwidth(140)
subset_input_form.set_fullparms(True)
cfg.set_config_value(
get_saved_save_input_id + "Parms",
[df_title, "./" + df_title + ".csv", df_title, "False"])
cfg.drop_config_value(get_saved_save_input_id + "ParmsProtect")
get_subset_input_html = subset_input_form.get_html()
get_subset_heading_html = "<div>Save Dataframe Subset</div><br>"
gridclasses = [
"dfc-top", "dfc-main", "dfcleanser-common-grid-header", "dfc-bottom"
]
gridhtmls = [
df_stats_table, df_cols_table, get_subset_heading_html,
get_subset_input_html
]
print("\n")
display_generic_grid("sw-utils-subset-run-wrapper", gridclasses, gridhtmls)
def clear_sw_utility_dfsubset_cfg_values() :
cfg.drop_config_value(swsw.get_subset_input_id+"Parms")
cfg.drop_config_value(swsw.get_subset_criteria_input_id+"Parms")
from dfcleanser.data_inspection.data_inspection_control import drop_working_df
drop_working_df()
swsm.Current_Manual_Step = 0
swsm.Current_Manual_Sequence = ""
return()
def display_scripting_forms():
from dfcleanser.scripting.data_scripting_control import get_current_scriptlog
if (not (get_current_scriptlog() == None)):
cfg.set_config_value(dc_script_input_id + "Parms",
[get_current_scriptlog()])
print("\n")
notes = []
notes.append("You can manually edit the script log as desired")
notes.append(
"The step numbers are cmmentary only and order is insignificant")
display_notes(notes)
script_form = InputForm(dc_script_input_id, dc_script_input_idList,
dc_script_input_labelList,
dc_script_input_typeList,
dc_script_input_placeholderList,
dc_script_input_jsList, dc_script_input_reqList)
script_form.set_shortForm(True)
script_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 140,
"left-margin": 70
})
script_form.set_gridwidth(880)
script_form.set_fullparms(True)
script_form_html = script_form.get_html()
script_title_html = "<div>Scripting</div><br>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"]
gridhtmls = [script_title_html, script_form_html]
from dfcleanser.common.common_utils import display_generic_grid
display_generic_grid("data-scripting-wrapper", gridclasses, gridhtmls)
cfg.drop_config_value(dc_script_input_id + "Parms")
print("\n")
def clear_data_scripting_cfg_values():
cfg.drop_config_value(cfg.SCRIPT_LOG_KEY)
cfg.drop_config_value(cfg.BACKUP_SCRIPT_LOG_KEY)
cfg.drop_config_value(cfg.SCRIPTING_FLAG_KEY)
return ()
def export_custom(parms):
"""
* --------------------------------------------------------------------------
* function : export a custom
*
* parms :
* parms - sql parms
*
* returns : N/A
* --------------------------------------------------------
"""
functionid = parms[0]
opstat = opStatus()
dispstats = False
if (functionid == 1):
opstat = process_custom_export(parms[1],
dew.custom_export_id,
display=True)
dispstats = True
elif (functionid == 2):
custom_code = "# custom export\n"
custom_code = custom_code + parms[1]
cfg.set_config_value(dew.custom_export_id + "Parms", custom_code)
display_export_forms(dem.EXPORT_CUSTOM_ONLY)
elif (functionid == 3):
cfg.drop_config_value(dew.custom_export_id + "Parms")
display_export_forms(dem.EXPORT_CUSTOM_ONLY)
elif (functionid == 5):
display_export_forms(dem.EXPORT_CUSTOM_ONLY, -1, True)
return (dispstats, opstat)
def display_data_inspection(option, parms=None):
"""
* --------------------------------------------------------------------------
* function : main data inspection processing
*
* parms :
* option - function option
* parms - associated parms
*
* returns :
* N/A
* --------------------------------------------------------
"""
from IPython.display import clear_output
clear_output()
opstat = opStatus()
from dfcleanser.common.html_widgets import define_inputs, are_owner_inputs_defined
if (not (are_owner_inputs_defined(cfg.DataInspection_ID))):
define_inputs(cfg.DataInspection_ID, diw.datainspection_inputs)
if (option == dim.MAIN_OPTION):
drop_working_df()
diw.display_dfc_inspection_main()
clear_data_inspection_data()
else:
diw.display_inspection_main_taskbar()
if (cfg.is_a_dfc_dataframe_loaded()):
if ((option == dim.DISPLAY_DATATYPES_OPTION)
or (option == dim.DISPLAY_NANS_OPTION)
or (option == dim.DISPLAY_ROWS_OPTION)
or (option == dim.DISPLAY_COLS_OPTION)
or (option == dim.DISPLAY_CATEGORIES_OPTION)):
fparms = get_parms_for_input(parms[0],
diw.data_inspection_df_input_idList)
if (len(fparms) > 0):
cfg.set_config_value(cfg.CURRENT_INSPECTION_DF, fparms[0])
if (not (option == dim.DISPLAY_ROWS_OPTION)):
drop_working_df()
if ((option == dim.DISPLAY_DATATYPES_OPTION)
or (option == dim.DISPLAY_FULL_COLUMN_NAMES)):
df_data_info = dim.get_df_datatypes_data(
cfg.get_current_chapter_df(cfg.DataInspection_ID))
display_inspect_datatypes(option, df_data_info)
elif (option == dim.DISPLAY_NANS_OPTION):
display_inspect_nans()
elif (option == dim.DISPLAY_ROWS_OPTION):
display_inspect_rows()
elif (option == dim.DISPLAY_COLS_OPTION):
if (len(parms) > 1):
display_inspect_cols(parms[1])
else:
display_inspect_cols(None)
elif (option == dim.DISPLAY_CATEGORIES_OPTION):
display_inspect_categories()
elif ((option == dim.DROP_ROW_NANS_OPTION)
or (option == dim.DROP_COL_NANS_OPTION)):
thresholdType = parms[0]
if (option == dim.DROP_ROW_NANS_OPTION):
fparms = get_parms_for_input(parms[1],
diw.drop_rows_input_idList)
else:
fparms = get_parms_for_input(parms[1],
diw.drop_columns_input_idList)
if (len(fparms) > 0):
try:
threshold = int(fparms[0])
except:
opstat.set_status(False)
if (option == dim.DROP_ROW_NANS_OPTION):
opstat.set_errorMsg("Drop Nan Rows Threshold value '" +
fparms[0] + "' is invalid")
else:
opstat.set_errorMsg("Drop Nan Cols Threshold value '" +
fparms[0] + "' is invalid")
threshold = None
else:
opstat.set_status(False)
if (option == dim.DROP_ROW_NANS_OPTION):
opstat.set_errorMsg(
"Drop Nan Rows Threshold value is not defined")
else:
opstat.set_errorMsg(
"Drop Nan Cols Threshold value is not defined")
threshold = None
if (option == dim.DROP_ROW_NANS_OPTION):
if (opstat.get_status()):
dropstats = drop_nan_rows(
cfg.get_current_chapter_df(cfg.DataInspection_ID),
threshold, thresholdType, opstat)
if (not (opstat.get_status())):
display_exception(opstat)
else:
if (dropstats[0] > 0):
display_status(
str(dropstats[0]) +
" Nan Rows Dropped Successfully")
else:
display_status(
"No Rows matching threshold were dropped")
else:
if (opstat.get_status()):
numcolsdropped = drop_nan_cols(
cfg.get_current_chapter_df(cfg.DataInspection_ID),
threshold, thresholdType, opstat)
if (not (opstat.get_status())):
display_exception(opstat)
else:
if (numcolsdropped > 0):
display_status(
str(numcolsdropped) +
" Columns with Nans Dropped Successfully")
else:
display_status(
" No Columns matching threshold were dropped")
elif (option == dim.DISPLAY_ROW_OPTION):
display_inspect_rows()
elif (option == dim.DISPLAY_COL_GRAPHS):
display_inspect_graphs(parms)
elif (option == dim.DISPLAY_COL_OUTLIERS):
display_inspect_outliers(parms[0])
elif (option == dim.DISPLAY_SCROLL_TO_DF_ROW):
diw.display_scroll_to_row()
elif (option == dim.PROCESS_SCROLL_TO_DF_ROW):
opstat = opStatus()
df = cfg.get_current_chapter_df(cfg.DataInspection_ID)
retparms = get_row_id_for_df(df, parms,
diw.scroll_df_rows_input_idList,
opstat)
if (opstat.get_status()):
if (retparms[1] == 0):
display_inspect_rows(retparms[0])
else:
display_inspect_rows(retparms[0])
else:
diw.display_scroll_to_row()
display_exception(opstat)
elif (option == dim.SCROLL_DF_ROWS_DOWN):
new_row_id = cfg.get_config_value(cfg.CURRENT_SCROLL_ROW_KEY)
if (new_row_id is None):
new_row_id = 0
else:
new_row_id = new_row_id + 200
df = cfg.get_current_chapter_df(cfg.DataInspection_ID)
if (new_row_id > len(df)):
new_row_id = cfg.get_config_value(
cfg.CURRENT_SCROLL_ROW_KEY)
display_inspect_rows(new_row_id)
elif (option == dim.SCROLL_DF_ROWS_UP):
new_row_id = cfg.get_config_value(cfg.CURRENT_SCROLL_ROW_KEY)
if (new_row_id is None):
new_row_id = 0
else:
new_row_id = new_row_id - 200
if (new_row_id < 0):
new_row_id = 0
display_inspect_rows(new_row_id)
elif (option == dim.DISPLAY_DF_ROW):
print("dim.DISPLAY_DF_ROW")
elif (option == dim.DISPLAY_DF_ROW_REMOTE):
chapterid = parms[0]
#print("chapterId",chapterid)
new_config_df = None
if (chapterid == cfg.DataInspection_ID):
new_config_df = cfg.get_config_value(cfg.CURRENT_INSPECTION_DF)
elif (chapterid == cfg.DataCleansing_ID):
new_config_df = cfg.get_config_value(cfg.CURRENT_CLEANSE_DF)
elif (chapterid == cfg.DataTransform_ID):
new_config_df = cfg.get_config_value(cfg.CURRENT_TRANSFORM_DF)
elif (chapterid == cfg.DataExport_ID):
new_config_df = cfg.get_config_value(cfg.CURRENT_EXPORT_DF)
elif (chapterid == cfg.DataImport_ID):
new_config_df = cfg.get_config_value(cfg.CURRENT_IMPORT_DF)
elif (chapterid == cfg.SWGeocodeUtility_ID):
new_config_df = cfg.get_config_value(cfg.CURRENT_GEOCODE_DF)
elif (chapterid == cfg.SWDFSubsetUtility_ID):
new_config_df = cfg.get_config_value(cfg.CURRENT_SUBSET_DF)
cfg.set_config_value(cfg.CURRENT_INSPECTION_DF, new_config_df)
display_inspect_rows()
else:
cfg.drop_config_value(cfg.CURRENT_INSPECTION_DF)
if (not (option == dim.MAIN_OPTION)):
cfg.display_no_dfs(cfg.DataInspection_ID)
from dfcleanser.common.display_utils import display_pop_up_buffer
display_pop_up_buffer()
def display_list_dict(stype, keyValue=None):
"""
* ------------------------------------------------------------------------
* function : display the dict or list start form
*
* parms :
* id - list or dict
* keyValue - dict or list name
*
* -------------------------------------------------------------------------
"""
#print("display_list_dict",stype,keyValue)
if (stype == swum.DICT_ID):
list_dict_form = build_dict_utility_input_form
else:
list_dict_form = build_list_utility_input_form
list_dict_input_form = InputForm(list_dict_form[0], list_dict_form[1],
list_dict_form[2], list_dict_form[3],
list_dict_form[4], list_dict_form[5],
list_dict_form[6])
selectDicts = []
if (stype == swum.DICT_ID):
from dfcleanser.sw_utilities.sw_utility_model import get_dicts_names, DFC_CREATED
dict_names = get_dicts_names(DFC_CREATED)
if (keyValue is None):
def_dict = dict_names[0]
else:
def_dict = keyValue
#print("def_dict",def_dict,keyValue)
dictssel = {
"default": def_dict,
"list": dict_names,
"callback": "select_dict"
}
seldict = swum.get_Dict(def_dict, DFC_CREATED)
keys = list(seldict.keys())
if ((def_dict == "Country_Codes") or (def_dict == "Language_Codes")):
keys.sort()
seldict = swum.get_pretty_dict(seldict, keys)
else:
from dfcleanser.sw_utilities.sw_utility_model import get_lists_names, DFC_CREATED
list_names = get_lists_names(DFC_CREATED)
if (keyValue is None):
def_list = list_names[0]
else:
def_list = keyValue
dictssel = {
"default": def_list,
"list": list_names,
"callback": "select_list"
}
sellist = str(swum.get_List(def_list, DFC_CREATED))
selectDicts.append(dictssel)
from dfcleanser.common.common_utils import get_select_defaults
get_select_defaults(list_dict_input_form, list_dict_form[0],
list_dict_form[1], list_dict_form[3], selectDicts)
list_dict_input_form.set_gridwidth(700)
list_dict_input_form.set_custombwidth(120)
list_dict_input_form.set_fullparms(True)
if (stype == swum.DICT_ID):
selparms = [def_dict, seldict]
cfg.set_config_value(build_dict_utility_input_id + "Parms", selparms)
cfg.set_config_value(build_dict_utility_input_id + "ParmsProtect",
[False, True])
help_note = "To retrieve a dict in python from dfcleanser.sw_utiliities.sw_utility.model call 'get_Dict(dictname)'.</br>To add a dict in python from dfcleanser.sw_utiliities.sw_utility.model call 'add_Dict(dictname,newdict)'"
else:
selparms = [def_list, sellist]
cfg.set_config_value(build_list_utility_input_id + "Parms", selparms)
cfg.set_config_value(build_list_utility_input_id + "ParmsProtect",
[False, True])
help_note = "To retrieve a list in python from dfcleanser.sw_utiliities.sw_utility.model call 'get_List(listname)'.</br>To add a list in python from dfcleanser.sw_utiliities.sw_utility.model call 'add_List(listtname,newlist)'"
from dfcleanser.common.common_utils import get_help_note_html
listdict_notes_html = get_help_note_html(help_note, 100, None, None)
list_dictcustom_html = ""
list_dictcustom_html = list_dict_input_form.get_html()
if (stype == swum.DICT_ID):
list_title_html = "<div>dfc Dicts</div><br></br>"
cfg.drop_config_value(build_dict_utility_input_id + "Parms")
else:
list_title_html = "<div>dfc Lists</div><br></br>"
cfg.drop_config_value(build_list_utility_input_id + "Parms")
gridclasses = ["dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"]
gridhtmls = [list_title_html, list_dictcustom_html, listdict_notes_html]
print("\n")
display_generic_grid("sw-utils-listdict-wrapper", gridclasses, gridhtmls)
def clear_system_cfg_values():
cfg.drop_config_value(sysw.dfmgr_input_id + "Parms")
cfg.drop_config_value(sysw.dfmgr_input_id + "ParmsProtect")
return (True)
def display_system_environment(funcId, parms=None):
"""
* --------------------------------------------------------------------------
* function : display system environment screens
*
* parms :
* funcId - display func id
* parms - associated parms
*
* returns :
* N/A
* --------------------------------------------------------
"""
if (not (cfg.check_if_dc_init())):
sysw.display_system_main_taskbar()
return
else:
from dfcleanser.common.html_widgets import define_inputs, are_owner_inputs_defined
if (not (are_owner_inputs_defined(cfg.System_ID))):
define_inputs(cfg.System_ID, sysw.system_inputs)
if (funcId == sysm.DISPLAY_MAIN):
display_main_tb()
clear_system_data()
if (not (funcId == sysm.PROCESS_EULA)):
if (not isEULA_read()):
#display_system_main_taskbar()
sysw.display_EULA()
return
if (funcId == sysm.DISPLAY_CHAPTERS):
sysw.display_system_chapters_taskbar()
if (funcId == sysm.RESET_CHAPTERS):
sysw.display_system_main_taskbar()
initialize_notebook()
if (funcId == sysm.PROCESS_CHAPTERS):
parms[0] = parms[0].replace("[", "")
parms[0] = parms[0].replace("]", "")
utils_cbs = parms[0].split(",")
utilscbs = []
for i in range(len(utils_cbs)):
if (utils_cbs[i] == '"True"'):
utilscbs.append(1)
else:
utilscbs.append(0)
from dfcleanser.system.load import reload_dfcleanser
reload_dfcleanser([utilscbs])
clear_cell()
sysw.display_system_main_taskbar()
elif (funcId == sysm.DISPLAY_DATAFRAMES):
if (not (parms is None)):
title = parms[0]
else:
title = None
sysw.display_system_main_taskbar()
sysw.display_df_dataframes(title)
elif (funcId == sysm.DISPLAY_ADD_DATAFRAME):
sysw.display_system_main_taskbar()
cfg.drop_config_value(sysw.dfmgr_add_input_id + "Parms")
sysw.display_add_df_input()
elif (funcId == sysm.PROCESS_DATAFRAME):
fid = parms[0]
from dfcleanser.common.common_utils import get_parms_for_input
fparms = get_parms_for_input(parms[1], sysw.dfmgr_input_idList)
dftitle = None
if (fid == sysm.DROP_DATAFRAME):
cfg.drop_dfc_dataframe(fparms[0])
elif (fid == sysm.SET_DATAFRAME):
print("sysm.SET_DATAFRAME")
elif (fid == sysm.UPDATE_DATAFRAME):
cfg.set_dfc_dataframe_notes(fparms[0], fparms[3])
dftitle = fparms[0]
elif (fid == sysm.RENAME_DATAFRAME):
cfg.rename_dfc_dataframe(
cfg.get_config_value(cfg.CURRENT_DF_DISPLAYED_KEY),
fparms[0])
dftitle = fparms[0]
sysw.display_system_main_taskbar()
sysw.display_df_dataframes(dftitle)
elif (funcId == sysm.PROCESS_ADD_DATAFRAME):
opstat = opStatus()
from dfcleanser.common.common_utils import get_parms_for_input
fparms = get_parms_for_input(parms[1], sysw.dfmgr_add_input_idList)
dftitle = fparms[0]
dfobject = fparms[1]
dfnotes = fparms[2]
if (not (len(dftitle)) > 0):
opstat.set_status(False)
opstat.set_errorMsg("Invalid df title parm")
else:
if (len(dfobject) > 0):
try:
add_df_js = ("add_new_dfc_df('" + dftitle + "', '" +
dfobject + "', '" + dfnotes + "');")
run_jscript(add_df_js, "fail to add dataframe : ")
except Exception:
opstat.set_status(False)
opstat.set_errorMsg(
"Unable to add df to dfc manager : " +
str(sys.exc_info()[0].__name__))
else:
opstat.set_status(False)
opstat.set_errorMsg("Invalid df name parm")
if (opstat.get_status()):
sysw.display_df_dataframes(dftitle)
else:
display_status(opstat.get_errorMsg())
sysw.display_add_df_input()
elif (funcId == sysm.DISPLAY_SYSTEM):
display_main_tb()
sysw.show_sys_info()
elif (funcId == sysm.DISPLAY_OFFLINE):
display_main_tb()
sysw.display_offline()
elif (funcId == sysm.DISPLAY_ABOUT):
sysw.display_system_main_taskbar()
sysw.show_about_info()
elif (funcId == sysm.DISPLAY_DFC_FILES):
sysw.display_system_main_taskbar()
sysw.display_dfc_files_form()
elif (funcId == sysm.DISPLAY_EULA):
display_main_tb()
sysw.display_EULA()
elif (funcId == sysm.DISPLAY_README):
display_main_tb()
sysw.display_README()
elif (funcId == sysm.PROCESS_EULA):
display_main_tb()
cfg.set_config_value(cfg.EULA_FLAG_KEY, "true")
elif (funcId == sysm.EXIT_SETUP):
from dfcleanser.system.load import unload_dfcleanser
unload_dfcleanser()
return
def process_sort_by_column(parms,display=True) :
"""
* --------------------------------------------------------------------------
* function : sort by column transform option
*
* parms :
* parms - associated parms
* display - display results flag
*
* returns :
* N/A
* --------------------------------------------------------
"""
opstat = opStatus()
fparms = get_parms_for_input(parms,dftw.sort_column_input_idList)
colname = fparms[0]
sortorder = fparms[1]
if(sortorder == "True") :
sortorder = True
else :
sortorder = False
sortkind = fparms[2]
sortkind = sortkind.lstrip("'")
sortkind = sortkind.rstrip("'")
naposition = fparms[3]
naposition = naposition.lstrip("'")
naposition = naposition.rstrip("'")
resetrowids = fparms[4]
if(resetrowids == "True") :
resetrowids = True
else :
resetrowids = False
if(opstat.get_status()) :
try :
df = cfg.get_current_chapter_df(cfg.DataTransform_ID)
df.sort_values(colname,axis=0,ascending=sortorder,inplace=True,kind=sortkind,na_position=naposition)
if(resetrowids) :
from dfcleanser.data_transform.data_transform_dataframe_control import reset_df_index
opstat = reset_df_index()
if(display) :
#make scriptable
add_to_script(["# sort by column ",
"from dfcleanser.data_transform.data_transform_columns_control import process_sort_by_column",
"process_sort_by_column(" + json.dumps(parms) + ",False)"],opstat)
opstat.set_errorMsg("df sorted by column '" + colname + "' successfully.")
except Exception as e:
opstat.store_exception("Sort df By Column Error : "+colname,e)
cfg.drop_config_value(dftw.sort_column_input_id+"Parms")
return(opstat)
def display_dfsubset_utility(optionId,parms=None) :
"""
* ---------------------------------------------------------
* function : main subset utility control
*
* parms :
* optionId - function to run
* parms - parms to ryn function
*
* returns :
* NA
* --------------------------------------------------------
"""
if(cfg.is_a_dfc_dataframe_loaded()) :
from IPython.display import clear_output
clear_output()
from dfcleanser.common.html_widgets import define_inputs, are_owner_inputs_defined
if(not (are_owner_inputs_defined(cfg.SWDFSubsetUtility_ID)) ) :
define_inputs(cfg.SWDFSubsetUtility_ID,swsw.SWUtility_subset_inputs)
if(optionId == swsm.DISPLAY_MAIN) :
swsw.get_dfsubset_main_taskbar()
clear_sw_utility_dfsubsetdata()
cfg.display_data_select_df(cfg.SWDFSubsetUtility_ID)
swsm.clear_current_subset_data()
elif(optionId == swsm.DISPLAY_GET_SUBSET) :
swsm.clear_current_subset_data()
if(DEBUG_SUBSET) :
swsm.set_current_subset_sequence(swsm.dfc_subset_sequence())
print("\ncurrent_subset_sequence\n")
print("input_df_title",swsm.get_current_subset_sequence().get_input_df_title())
print("get_sequence_title",swsm.get_current_subset_sequence().get_sequence_title())
print("get_sequence_steps",swsm.get_current_subset_sequence().get_sequence_steps())
if(not (swsm.get_current_subset_sequence().get_sequence_steps() is None)) :
print("get_total_sequence_steps",swsm.get_current_subset_sequence().get_total_sequence_steps())
print("get_output_csv",swsm.get_current_subset_sequence().get_output_csv())
print("get_output_dfc_df_title",swsm.get_current_subset_sequence().get_output_dfc_df_title())
swsw.display_df_subset_setup()
if(DEBUG_SUBSET) :
print("DISPLAY_GET_SUBSET",parms)
print("DISPLAY_GET_SUBSET : clear data")
print(swsm.get_current_subset_sequence())
print(swsm.get_current_subset_df())
print(swsm.get_current_subset_step())
print("new_sequence")
swsm.dump_current_step()
swsm.dump_current_sequence()
elif(optionId == swsm.PROCESS_GET_SUBSET) :
current_step = swsm.get_current_subset_step()
if(not (current_step is None)) :
current_sequence = swsm.get_current_subset_sequence()
current_sequence.add_step_to_sequence_steps(current_step)
fparms = get_parms_for_input(parms,swsw.get_subset_input_idList)
if(len(fparms) > 0) :
df_title = fparms[0]
df = cfg.get_dfc_dataframe_df(df_title)
col_names = fparms[1]
col_action = fparms[3]
new_subset_df = drop_add_cols(col_names,col_action,df)
new_subset_df_title = df_title
new_subset_step = swsm.dfc_subset_step(new_subset_df_title,col_names,col_action)
swsm.set_current_subset_step(new_subset_step)
swsm.set_current_subset_df(new_subset_df)
swsw.display_df_criteria(new_subset_df_title,new_subset_df)
if(DEBUG_SUBSET) :
print("\nPROCESS_GET_SUBSET\n ",parms,"\n ",fparms)
swsm.dump_current_step()
swsm.dump_current_sequence()
elif(optionId == swsm.DISPLAY_SAVED_SUBSET) :
swsw.display_saved_subset_sequences()
if(DEBUG_SUBSET) :
print("\nDISPLAY_SAVED_SUBSET",parms)
elif(optionId == swsm.PROCESS_RUN_CRITERIA) :
opstat = opStatus()
fparms = get_parms_for_input(parms,swsw.get_subset_criteria_input_idList)
subset_title = fparms[0]
if(len(subset_title) == 0) :
current_sequence = swsm.get_current_subset_sequence()
total_steps = current_sequence.get_total_sequence_steps()
current_step = swsm.get_current_subset_step()
subset_title = current_step.get_input_subset_df_title() + "_subset_" + str(total_steps+1)
criteria = fparms[2]
if(len(criteria) > 0) :
try :
clock = RunningClock()
clock.start()
final_criteria = (swsm.starting_criteria_preamble + criteria + swsm.starting_criteria_postamble)
exec(final_criteria)
current_step = swsm.get_current_subset_step()
current_step.set_criteria(criteria)
current_step.set_output_subset_df_title(subset_title)
clock.stop()
except Exception as e:
opstat.store_exception("Error running df_criteria " + criteria,e)
clock.stop()
if(opstat.get_status()) :
swsw.display_process_subset()
else :
display_exception(opstat)
if(DEBUG_SUBSET) :
print("PROCESS_RUN_CRITERIA : End")
swsm.dump_current_step()
swsm.dump_current_sequence()
elif(optionId == swsm.DISPLAY_SAVE_SUBSET) :
fparms = get_parms_for_input(parms,swsw.get_subset_run_input_idList)
current_sequence = swsm.get_current_subset_sequence()
current_step = swsm.get_current_subset_step()
current_sequence.add_step_to_sequence_steps(current_step)
if(len(fparms) > 0) :
df_title = fparms[0]
df = swsm.get_current_subset_df()
col_names = fparms[1]
col_action = fparms[3]
new_subset_df = drop_add_cols(col_names,col_action,df)
swsw.display_save_subset(df_title,new_subset_df)
if(DEBUG_SUBSET) :
print("DISPLAY_SAVE_SUBSET",parms,fparms)
swsm.dump_current_step()
swsm.dump_current_sequence()
elif(optionId == swsm.DISPLAY_SAVE_AND_GET_SUBSET) :
fparms = get_parms_for_input(parms,swsw.get_subset_run_input_idList)
current_sequence = swsm.get_current_subset_sequence()
current_step = swsm.get_current_subset_step()
current_sequence.add_step_to_sequence_steps(current_step)
if(len(fparms) > 0) :
df_title = fparms[0]
col_names = fparms[1]
col_action = fparms[3]
new_subset_step = swsm.dfc_subset_step(df_title,col_names,col_action)
df = swsm.get_current_subset_df()
new_subset_df = drop_add_cols(col_names,col_action,df)
swsm.set_current_subset_df(new_subset_df)
swsm.set_current_subset_step(new_subset_step)
swsw.display_df_criteria(df_title,new_subset_df)
if(DEBUG_SUBSET) :
print("PROCESS_SAVE_AND_GET_SUBSET",parms,fparms)
swsm.dump_current_step()
swsm.dump_current_sequence()
elif(optionId == swsm.PROCESS_SAVE_SUBSET) :
save_subset_run(parms,0)
elif(optionId == swsm.PROCESS_SUBSET_SEQUENCE) :
opstat = opStatus()
fparms = get_parms_for_input(parms,swsw.get_subset_sequences_input_idList)
sequence = fparms[0]
run_option = fparms[1]
saved_sequence = swsm.get_subset_sequence(sequence)
first_step = saved_sequence.get_sequence_step(0)
df_title = first_step.get_input_subset_df_title()
df = cfg.get_dfc_dataframe_df(df_title)
if(df is None) :
swsw.get_dfsubset_main_taskbar()
cfg.display_data_select_df(cfg.SWDFSubsetUtility_ID)
opstat.set_status(False)
opstat.set_errorMsg("subset sequence starting df '" + df_title + "' is not currently loaded in dfc")
display_exception(opstat)
else :
if(run_option == "Auto Run") :
total_steps = saved_sequence.get_total_sequence_steps()
swsm.set_current_subset_df(df)
for i in range(total_steps) :
current_step = saved_sequence.get_sequence_step(i)
current_columns = current_step.get_col_names_list()
columns_action = current_step.get_keep_drop_flag()
criteria = current_step.get_criteria()
output_df_title = current_step.get_output_subset_df_title()
current_df = swsm.get_current_subset_df()
if(len(current_columns) > 0) :
colnames = list(current_df.columns)
drop_columns = []
for i in range(len(colnames)) :
if(columns_action == "Keep") :
if(not (colnames[i] in current_columns)) :
drop_columns.append(colnames[i])
else :
if(colnames[i] in current_columns) :
drop_columns.append(colnames[i])
if(len(drop_columns) > 0 ) :
try :
current_df.drop(drop_columns, axis=1, inplace=True)
except :
opstat.set_status(False)
opstat.set_errorMsg("Unable to drop columns from subset dataframe")
swsm.set_current_subset_df(current_df)
try :
current_df = swsm.get_current_subset_df()
exec(criteria + swsm.starting_criteria_postamble)
current_df = swsm.get_current_subset_df()
except Exception as e:
opstat.store_exception("Error running subset sequence '" + sequence + "'",e)
swsw.display_save_subset(output_df_title,swsm.get_current_subset_df(),True)
else :
total_steps = saved_sequence.get_total_sequence_steps()
swsm.set_current_subset_df(df)
current_step = saved_sequence.get_sequence_step(0)
current_df_title = current_step.get_input_subset_df_title()
current_columns = current_step.get_col_names_list()
columns_action = current_step.get_keep_drop_flag()
swsw.display_manual_df_subset_setup(saved_sequence.get_sequence_title(),current_df_title,current_columns,columns_action,0)
swsm.set_current_subset_step_id(0)
swsm.set_current_subset_sequence(saved_sequence)
elif(optionId == swsm.PROCESS_GET_NEXT_SUBSET) :
fparms = get_parms_for_input(parms,swsw.get_manual_input_idList)
collist = fparms[1]
collist = collist.lstrip("[")
collist = collist.rstrip("]")
collist = collist.split(",")
keep_drop = fparms[3]
saved_sequence = swsm.get_current_subset_sequence()
total_steps = saved_sequence.get_total_sequence_steps()
current_step_id = swsm.get_current_subset_step_id()
if(current_step_id < total_steps) :
current_step = saved_sequence.get_sequence_step(swsm.get_current_subset_step_id())
else :
swsm.set_current_subset_step_col_names_list(collist)
swsm.set_current_subset_keep_drop_flag(keep_drop)
current_step = swsm.get_current_subset_step()
swsm.dump_current_step()
swsm.set_current_subset_step(current_step)
current_df_title = current_step.get_input_subset_df_title()
current_df = swsm.get_current_subset_df()
current_columns = current_step.get_col_names_list()
columns_action = current_step.get_keep_drop_flag()
criteria = current_step.get_criteria()
output_df_title = current_step.get_output_subset_df_title()
if(len(current_columns) > 0) :
colnames = list(current_df.columns)
drop_columns = []
for i in range(len(colnames)) :
if(columns_action == "Keep") :
if(not (colnames[i] in current_columns)) :
drop_columns.append(colnames[i])
else :
if(colnames[i] in current_columns) :
drop_columns.append(colnames[i])
if(len(drop_columns) > 0 ) :
try :
current_df.drop(drop_columns, axis=1, inplace=True)
except :
opstat.set_status(False)
opstat.set_errorMsg("Unable to drop columns from subset dataframe")
swsw.display_next_criteria(current_df_title,current_df,criteria,output_df_title)
elif(optionId == swsm.PROCESS_NEXT_CRITERIA) :
opstat = opStatus()
fparms = get_parms_for_input(parms,swsw.get_next_criteria_input_idList)
output_df_title = fparms[0]
criteria = fparms[2]
current_sequence = swsm.get_subset_sequence(sequence)
sequence_title = current_sequence.get_sequence_title()
try :
current_df = swsm.get_current_subset_df()
exec(criteria + swsm.starting_criteria_postamble)
current_df = swsm.get_current_subset_df()
except Exception as e:
opstat.store_exception("Error running subset sequence '" + sequence_title + "'",e)
current_df_title = current_step.get_input_subset_df_title()
current_df = swsm.get_current_subset_df()
criteria = current_step.get_criteria()
output_df_title = current_step.get_output_subset_df_title()
swsw.display_next_criteria(current_df_title,current_df,criteria,output_df_title)
display_exception(opstat)
if(opstat.get_status()) :
swsm.set_current_subset_df(current_df)
swsm.set_current_subset_step_id(swsm.get_current_subset_step_id() + 1)
if(swsm.get_current_subset_step_id() >= swsm.get_current_subset_sequence().get_total_sequence_steps()) :
swsw.display_sequence_save_subset(output_df_title,swsm.get_current_subset_df())
else :
current_step = swsm.get_current_subset_sequence().get_sequence_step(swsm.get_current_subset_step_id())
current_df_title = current_step.get_input_subset_df_title()
current_columns = current_step.get_col_names_list()
columns_action = current_step.get_keep_drop_flag()
swsw.display_manual_df_subset_setup(swsm.get_current_subset_sequence().get_sequence_title(),current_df_title,current_columns,columns_action,swsm.get_current_subset_step_id())
elif(optionId == swsm.DISPLAY_NEW_STEP) :
current_sequence = swsm.get_current_subset_sequence()
sequence_title = current_sequence.get_sequence_title()
current_step = swsm.get_current_subset_step()
df_title = current_step.get_output_subset_df_title()
current_df = swsm.get_current_subset_df()
current_columns = []
current_action = "Keep"
criteria = swsm.starting_criteria
output_df_title = ""
current_step = swsm.dfc_subset_step(df_title,current_columns,current_action,criteria,output_df_title)
swsm.set_current_subset_step(current_step)
swsw.display_manual_df_subset_setup(sequence_title,df_title,current_columns,current_action,swsm.get_current_subset_step_id())
#swsw.display_next_criteria(df_title,current_df,criteria,output_df_title)
elif(optionId == swsm.PROCESS_SAVE_SAVED_SUBSET ) :
save_subset_run(parms,1)
elif(optionId == swsm.DISPLAY_GET_REMOTE_SUBSET) :
chapterid = parms[0]
new_config_df = None
if(chapterid == cfg.DataInspection_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_INSPECTION_DF)
elif(chapterid == cfg.DataCleansing_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_CLEANSE_DF)
elif(chapterid == cfg.DataTransform_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_TRANSFORM_DF)
elif(chapterid == cfg.DataExport_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_EXPORT_DF)
elif(chapterid == cfg.DataImport_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_IMPORT_DF)
elif(chapterid == cfg.SWGeocodeUtility_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_GEOCODE_DF)
elif(chapterid == cfg.SWDFSubsetUtility_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_SUBSET_DF)
cfg.set_config_value(cfg.CURRENT_SUBSET_DF,new_config_df)
swsm.clear_current_subset_data()
swsw.display_df_subset_setup()
else :
swsw.get_dfsubset_main_taskbar()
cfg.drop_config_value(cfg.CURRENT_SUBSET_DF)
clear_sw_utility_dfsubsetdata()
cfg.display_data_select_df(cfg.SWDFSubsetUtility_ID)
if(not(optionId == swsm.DISPLAY_MAIN)) :
cfg.display_no_dfs(cfg.SWDFSubsetUtility_ID)
def clear_data_export_cfg_values():
cfg.drop_config_value(cfg.CURRENT_EXPORT_START_TIME)
return (True)
def display_dc_export_forms(exid, detid=0, notes=False):
"""
* --------------------------------------------------------------------------
* function : display pandas export input forms
*
* parms :
* exid - export type
* detid - detail id
* notes - notes flag
*
* returns : N/A
* --------------------------------------------------------
"""
clear_output()
# add the main import task bar
if (exid == dem.EXPORT_TB_ONLY):
display_export_main_taskbar()
cfg.display_data_select_df(cfg.DataExport_ID)
else:
if (cfg.is_a_dfc_dataframe_loaded()):
# add the pandas import task bar or pandas details form
if ((exid == dem.EXPORT_PANDAS_TB_ONLY)
or (exid == dem.EXPORT_PANDAS_TB_PLUS_DETAILS)):
# add the pandas export details form
if (exid == dem.EXPORT_PANDAS_TB_PLUS_DETAILS):
if (detid == dem.SQLTABLE_EXPORT):
import dfcleanser.common.db_utils as dbutils
cfg.drop_config_value(pandas_export_sqltable_id +
"Parms")
dbid = cfg.get_config_value(cfg.CURRENT_DB_ID_KEY)
if (dbid == None):
cfg.set_config_value(cfg.CURRENT_DB_ID_KEY,
dbutils.MySql)
conparms = cfg.get_config_value(
dbutils.MYSQL_DBCON_PARMS)
if (conparms == None):
conparms = [
"", "", "", "", dbutils.pymysql_library
]
elif (dbid == dbutils.MySql):
conparms = cfg.get_config_value(
dbutils.MYSQL_DBCON_PARMS)
if (conparms == None):
conparms = [
"", "", "", "", dbutils.pymysql_library
]
elif (dbid == dbutils.MS_SQL_Server):
conparms = cfg.get_config_value(
dbutils.MSSQL_DBCON_PARMS)
if (conparms == None):
conparms = [
"", "", "", "", dbutils.pyodbc_library
]
elif (dbid == dbutils.SQLite):
conparms = cfg.get_config_value(
dbutils.SQLITE_DBCON_PARMS)
if (conparms == None):
conparms = ["", dbutils.sqlite3_library]
elif (dbid == dbutils.Postgresql):
conparms = cfg.get_config_value(
dbutils.POSTGRESQL_DBCON_PARMS)
if (conparms == None):
conparms = [
"", "", "", "", dbutils.psycopg2_library
]
elif (dbid == dbutils.Oracle):
conparms = cfg.get_config_value(
dbutils.ORACLE_DBCON_PARMS)
if (conparms == None):
conparms = [
"", "", "", dbutils.cx_oracle_library
]
elif (dbid == dbutils.Custom):
conparms = cfg.get_config_value(
dbutils.CUSTOM_DBCON_PARMS)
if (conparms == None): conparms = [""]
dbutils.display_db_connector_inputs(
cfg.get_config_value(cfg.CURRENT_DB_ID_KEY),
conparms, dbutils.SQL_EXPORT)
else:
display_export_main_taskbar()
pandas_export_form = get_pandas_export_input_form(
detid)
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
pandas_export_form.set_shortForm(True)
pandas_export_form.set_gridwidth(640)
pandas_export_form.set_custombwidth(110)
else:
pandas_export_form.set_gridwidth(480)
pandas_export_form.set_custombwidth(100)
pandas_input_html = ""
pandas_input_html = pandas_export_form.get_html()
pandas_input_heading_html = "<div>" + get_pandas_export_input_title(
detid) + "</div>"
gridclasses = [
"dfcleanser-common-grid-header", "dfc-footer"
]
gridhtmls = [
pandas_input_heading_html, pandas_input_html
]
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
display_generic_grid("data-import-wrapper",
gridclasses, gridhtmls)
else:
display_generic_grid("data-import-pop-up-wrapper",
gridclasses, gridhtmls)
else:
display_pandas_export_taskbar()
elif (exid == dem.EXPORT_CUSTOM_ONLY):
print("dem.EXPORT_PANDAS_TB_ONLY")
# add the custom import form
exportCustomDetailsForm = InputForm(
custom_export_id, custom_export_idList,
custom_export_labelList, custom_export_typeList,
custom_export_placeholderList, custom_export_jsList,
custom_export_reqList)
if (notes):
customNotes = [
"To create custom export code in the code cell below hit 'New Custom Export'",
" (enter and test export in the code cell below)",
" (leave the '# custom export' comment line in the code cell",
" (call dfcleanser.common.cfg.get_dfc_dataframe_df() to get the current dataframe)",
"To run the export code in the Custom Export Code box hit 'Run Custom Export' button",
" (only the code in the Custom Export Code box is run and stored for scripting)",
"Once import successful hit 'Save Custom Import' button to store import code for future retrieval",
"To drop the custom export code and clear the Custom Export Code box hit 'Drop Custom Export' button"
]
print("\n")
display_inline_help(customNotes, 92)
selectDicts = []
df_list = cfg.get_dfc_dataframes_select_list(cfg.DataExport_ID)
selectDicts.append(df_list)
flags = {"default": "False", "list": ["True", "False"]}
selectDicts.append(flags)
get_select_defaults(exportCustomDetailsForm, custom_export_id,
custom_export_idList,
custom_export_typeList, selectDicts)
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
exportCustomDetailsForm.set_shortForm(True)
exportCustomDetailsForm.set_gridwidth(640)
exportCustomDetailsForm.set_custombwidth(110)
else:
exportCustomDetailsForm.set_gridwidth(480)
exportCustomDetailsForm.set_custombwidth(100)
exportCustomDetailsForm.set_fullparms(True)
from dfcleanser.common.html_widgets import new_line
custom_code = "# add USER CODE to export the df" + new_line + new_line
custom_code = (
custom_code +
"from dfcleanser.common.cfg import get_dfc_dataframe" +
new_line)
custom_code = (custom_code +
"df = get_dfc_dataframe_df(dataframe_title)" +
new_line + new_line)
custom_code = (custom_code + "# USER CODE" + new_line)
cfg.set_config_value(custom_export_id + "Parms",
["", custom_code, ""])
custom_export_html = ""
custom_export_html = exportCustomDetailsForm.get_html()
custom_export_heading_html = "<div>Custom Export</div><br>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"]
gridhtmls = [custom_export_heading_html, custom_export_html]
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
display_generic_grid("data-import-wrapper", gridclasses,
gridhtmls)
else:
display_generic_grid("data-import-pop-up-wrapper",
gridclasses, gridhtmls)
else:
display_export_main_taskbar()
cfg.display_data_select_df(cfg.DataExport_ID)
cfg.display_no_dfs(cfg.DataExport_ID)
def display_dict_maint(keyValue=None, loadfile=None):
"""
* ------------------------------------------------------------------------
* function : display the user dicts maintenance form
*
* parms :
* keyValue - dict name
*
* -------------------------------------------------------------------------
"""
#print("display_dict_maint",keyValue,loadfile)
opstat = opStatus()
if (loadfile is None):
dict_maint_input_form = InputForm(
maint_dict_utility_input_id, maint_dict_utility_input_idList,
maint_dict_utility_input_labelList,
maint_dict_utility_input_typeList,
maint_dict_utility_input_placeholderList,
maint_dict_utility_input_jsList, maint_dict_utility_input_reqList)
else:
dict_maint_input_form = InputForm(
maint_dict_file_utility_input_id,
maint_dict_file_utility_input_idList,
maint_dict_file_utility_input_labelList,
maint_dict_file_utility_input_typeList,
maint_dict_file_utility_input_placeholderList,
maint_dict_file_utility_input_jsList,
maint_dict_file_utility_input_reqList)
selectDicts = []
from dfcleanser.sw_utilities.sw_utility_model import get_dicts_names, USER_CREATED
dict_names = get_dicts_names(USER_CREATED)
#print("dict_names",dict_names)
if (not (dict_names is None)):
if (keyValue is None):
def_dict = dict_names[0]
else:
def_dict = keyValue
seldict = swum.get_Dict(def_dict, USER_CREATED)
keys = list(seldict.keys())
if ((def_dict == "Country_Codes") or (def_dict == "Language_Codes")):
keys.sort()
seldict = swum.get_pretty_dict(seldict, keys)
else:
dict_names = ["No User dicts defined"]
def_dict = "No User dicts defined"
seldict = "User defined dict"
dictssel = {
"default": def_dict,
"list": dict_names,
"callback": "select_dict"
}
selectDicts.append(dictssel)
from dfcleanser.common.common_utils import get_select_defaults
get_select_defaults(dict_maint_input_form, maint_dict_utility_input_id,
maint_dict_utility_input_idList,
maint_dict_utility_input_typeList, selectDicts)
dict_maint_input_form.set_gridwidth(700)
#dict_maint_input_form.set_custombwidth(110)
if (loadfile is None):
dict_maint_input_form.set_buttonstyle({
"font-size": 13,
"height": 75,
"width": 90,
"left-margin": 20
})
else:
dict_maint_input_form.set_buttonstyle({
"font-size": 13,
"height": 75,
"width": 90,
"left-margin": 205
})
dict_maint_input_form.set_fullparms(True)
cfg.drop_config_value(maint_dict_utility_input_id + "Parms")
cfg.drop_config_value(maint_dict_utility_input_id + "ParmsProtect")
if (not (loadfile is None)):
import json
#from dfcleanser.common.common_utils import does_file_exist
#print("does_file_exist",does_file_exist(loadfile))
try:
with open(loadfile, 'r') as ds_file:
ds = json.load(ds_file)
ds_file.close()
keys = list(ds.keys())
seldict = swum.get_pretty_dict(ds, keys)
#print(seldict)
except Exception as e:
opstat.set_errorMsg("invalid user file to load " + loadfile)
opstat.set_exception(e)
if (opstat.get_status()):
if (loadfile is None):
cfg.set_config_value(maint_dict_utility_input_id + "Parms",
[def_dict, "", seldict, ""])
else:
cfg.set_config_value(maint_dict_utility_input_id + "Parms",
[def_dict, "", seldict, loadfile])
cfg.set_config_value(maint_dict_utility_input_id + "ParmsProtect",
[True, False, True, True])
help_note = "To add a user dict enter parms and values above and click on 'Add User Dict'.</br>To update the current dict change values and click on 'Update User Dict'"
from dfcleanser.common.common_utils import get_help_note_html
dict_maint_notes_html = get_help_note_html(help_note, 80, 75, None)
dict_maint_html = "Fill in new user dict parms or update currently displayed user dict."
dict_maint_html = dict_maint_input_form.get_html()
dict_maint_title_html = "<div>User Dicts</div><br></br>"
gridclasses = [
"dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"
]
gridhtmls = [
dict_maint_title_html, dict_maint_html, dict_maint_notes_html
]
#print(dict_maint_html)
#print(dict_maint_notes_html)
print("\n")
display_generic_grid("sw-utils-listdict-wrapper", gridclasses,
gridhtmls)
else:
display_exception(opstat)
add_error_to_log("[Get User Dict from File] " + loadfile +
str(sys.exc_info()[0].__name__))
def drop_data_export_start():
cfg.drop_config_value(cfg.CURRENT_EXPORT_START_TIME)
def clear_sw_utility_cfg_values():
cfg.drop_config_value(cfg.CURRENT_GENERIC_FUNCTION)
return
def display_list_maint(keyValue=None, loadfile=None):
"""
* ------------------------------------------------------------------------
* function : display the user lists maintenance form
*
* parms :
* keyValue - list name
*
* -------------------------------------------------------------------------
"""
opstat = opStatus()
if (loadfile is None):
list_maint_input_form = InputForm(
maint_list_utility_input_id, maint_list_utility_input_idList,
maint_list_utility_input_labelList,
maint_list_utility_input_typeList,
maint_list_utility_input_placeholderList,
maint_list_utility_input_jsList, maint_list_utility_input_reqList)
else:
list_maint_input_form = InputForm(
maint_list_file_utility_input_id,
maint_list_file_utility_input_idList,
maint_list_file_utility_input_labelList,
maint_list_file_utility_input_typeList,
maint_list_file_utility_input_placeholderList,
maint_list_file_utility_input_jsList,
maint_list_file_utility_input_reqList)
selectDicts = []
from dfcleanser.sw_utilities.sw_utility_model import get_lists_names, USER_CREATED
list_names = get_lists_names(USER_CREATED)
#print("list_names",list_names)
if (not (list_names is None)):
if (keyValue is None):
def_list = list_names[0]
else:
def_list = keyValue
sellist = swum.get_List(def_list, USER_CREATED)
dsstr = "["
for i in range(len(sellist)):
dsstr = dsstr + str(sellist[i])
if (i == (len(sellist) - 1)):
dsstr = dsstr + "]"
else:
dsstr = dsstr + ","
else:
list_names = ["No User lists defined"]
def_list = "No User lists defined"
sellist = "User defined list"
listssel = {
"default": def_list,
"list": list_names,
"callback": "select_list"
}
selectDicts.append(listssel)
from dfcleanser.common.common_utils import get_select_defaults
get_select_defaults(list_maint_input_form, maint_list_utility_input_id,
maint_list_utility_input_idList,
maint_list_utility_input_typeList, selectDicts)
list_maint_input_form.set_gridwidth(700)
if (loadfile is None):
list_maint_input_form.set_buttonstyle({
"font-size": 13,
"height": 75,
"width": 90,
"left-margin": 20
})
else:
list_maint_input_form.set_buttonstyle({
"font-size": 13,
"height": 75,
"width": 90,
"left-margin": 205
})
list_maint_input_form.set_fullparms(True)
cfg.drop_config_value(maint_list_utility_input_id + "Parms")
cfg.drop_config_value(maint_list_utility_input_id + "ParmsProtect")
if (not (loadfile is None)):
import json
from dfcleanser.common.common_utils import does_file_exist
if (does_file_exist(loadfile)):
try:
with open(loadfile, 'r') as ds_file:
ds = json.load(ds_file)
ds_file.close()
dsstr = "["
for i in range(len(ds)):
dsstr = dsstr + str(ds[i])
if (i == (len(ds) - 1)):
dsstr = dsstr + "]"
else:
dsstr = dsstr + ","
except Exception as e:
opstat.set_status(False)
opstat.set_errorMsg("Error processing user file to load" +
loadfile)
opstat.set_exception(e)
else:
opstat.set_status(False)
opstat.set_errorMsg("invalid user file to load" + loadfile)
if (opstat.get_status()):
if (loadfile is None):
cfg.set_config_value(maint_list_utility_input_id + "Parms",
[def_list, "", dsstr, ""])
else:
cfg.set_config_value(maint_list_utility_input_id + "Parms",
[def_list, "", dsstr, loadfile])
cfg.set_config_value(maint_list_utility_input_id + "ParmsProtect",
[True, False, True, True])
help_note = "To add a user list enter parms and values above and click on 'Add User List'.</br>To update the current list change values and click on 'Update User List'"
from dfcleanser.common.common_utils import get_help_note_html
list_maint_notes_html = get_help_note_html(help_note, 80, 75, None)
list_maint_html = ""
list_maint_html = list_maint_input_form.get_html()
list_maint_title_html = "<div>User Lists</div><br></br>"
gridclasses = [
"dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"
]
gridhtmls = [
list_maint_title_html, list_maint_html, list_maint_notes_html
]
#print(list_maint_html)
#print(list_maint_notes_html)
print("\n")
display_generic_grid("sw-utils-listdict-wrapper", gridclasses,
gridhtmls)
else:
display_exception(opstat)
add_error_to_log("[Get User Dict from File] " + loadfile +
str(sys.exc_info()[0].__name__))
def display_export_dc_sql_details_forms(dblibid):
"""
* --------------------------------------------------------------------------
* function : display export sql form
*
* parms :
* dblibid - db loib id
*
* returns : N/A
* --------------------------------------------------------
"""
import dfcleanser.common.db_utils as dbutils
if (dblibid == None):
cfg.drop_config_value(cfg.CURRENT_DB_ID_KEY)
if ((dblibid == dbutils.pymysql_library)
or (dblibid == dbutils.mysql_connector_library)):
cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.MySql)
elif ((dblibid == dbutils.pyodbc_library)
or (dblibid == dbutils.pymssql_library)):
cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.MS_SQL_Server)
elif ((dblibid == dbutils.sqlite_library)
or (dblibid == dbutils.sqlite3_library)):
cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.SQLite)
elif (dblibid == dbutils.psycopg2_library):
cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.Postgresql)
elif (dblibid == dbutils.cx_oracle_library):
cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.Oracle)
elif (dblibid == "custom"):
cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.Custom)
dbid = cfg.get_config_value(cfg.CURRENT_DB_ID_KEY)
if (dbid == None):
inparms = cfg.get_config_value(cfg.MYSQL_IMPORT_PARMS_KEY)
if (inparms == None):
inparms = ["", "", "", "", dblibid]
else:
inparms[4] = dblibid
if (dbid == dbutils.MySql):
inparms = cfg.get_config_value(cfg.MYSQL_IMPORT_PARMS_KEY)
if (inparms == None):
inparms = ["", "", "", "", dblibid]
else:
inparms[4] = dblibid
elif (dbid == dbutils.MS_SQL_Server):
inparms = cfg.get_config_value(cfg.MSSQL_IMPORT_PARMS_KEY)
if (inparms == None):
inparms = ["", "", "", "", dblibid]
else:
inparms[4] = dblibid
elif (dbid == dbutils.Postgresql):
inparms = cfg.get_config_value(cfg.POSTGRESQL_IMPORT_PARMS_KEY)
if (inparms == None):
inparms = ["", "", "", "", dblibid]
else:
inparms[4] = dblibid
elif (dbid == dbutils.SQLite):
inparms = cfg.get_config_value(cfg.SQLITE_IMPORT_PARMS_KEY)
if (inparms == None):
inparms = ["", dblibid]
else:
inparms[1] = dblibid
elif (dbid == dbutils.Oracle):
inparms = cfg.get_config_value(cfg.ORACLE_IMPORT_PARMS_KEY)
if (inparms == None):
inparms = ["", "", "", dblibid]
else:
inparms[1] = dblibid
elif (dbid == dbutils.Custom):
inparms = cfg.get_config_value(cfg.CUSTOM_IMPORT_PARMS_KEY)
from dfcleanser.common.db_utils import display_db_connector_inputs, SQL_EXPORT
display_db_connector_inputs(cfg.get_config_value(cfg.CURRENT_DB_ID_KEY),
inparms, SQL_EXPORT)
