def display_scripting_forms():
from dfcleanser.scripting.data_scripting_control import get_current_scriptlog
if (not (get_current_scriptlog() == None)):
cfg.set_config_value(dc_script_input_id + "Parms",
[get_current_scriptlog()])
print("\n")
notes = []
notes.append("You can manually edit the script log as desired")
notes.append(
"The step numbers are cmmentary only and order is insignificant")
display_notes(notes)
script_form = InputForm(dc_script_input_id, dc_script_input_idList,
dc_script_input_labelList,
dc_script_input_typeList,
dc_script_input_placeholderList,
dc_script_input_jsList, dc_script_input_reqList)
script_form.set_shortForm(True)
script_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 140,
"left-margin": 70
})
script_form.set_gridwidth(880)
script_form.set_fullparms(True)
script_form_html = script_form.get_html()
script_title_html = "<div>Scripting</div><br>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"]
gridhtmls = [script_title_html, script_form_html]
from dfcleanser.common.common_utils import display_generic_grid
display_generic_grid("data-scripting-wrapper", gridclasses, gridhtmls)
cfg.drop_config_value(dc_script_input_id + "Parms")
print("\n")
def display_list_maint(keyValue=None, loadfile=None):
"""
* ------------------------------------------------------------------------
* function : display the user lists maintenance form
*
* parms :
* keyValue - list name
*
* -------------------------------------------------------------------------
"""
opstat = opStatus()
if (loadfile is None):
list_maint_input_form = InputForm(
maint_list_utility_input_id, maint_list_utility_input_idList,
maint_list_utility_input_labelList,
maint_list_utility_input_typeList,
maint_list_utility_input_placeholderList,
maint_list_utility_input_jsList, maint_list_utility_input_reqList)
else:
list_maint_input_form = InputForm(
maint_list_file_utility_input_id,
maint_list_file_utility_input_idList,
maint_list_file_utility_input_labelList,
maint_list_file_utility_input_typeList,
maint_list_file_utility_input_placeholderList,
maint_list_file_utility_input_jsList,
maint_list_file_utility_input_reqList)
selectDicts = []
from dfcleanser.sw_utilities.sw_utility_model import get_lists_names, USER_CREATED
list_names = get_lists_names(USER_CREATED)
#print("list_names",list_names)
if (not (list_names is None)):
if (keyValue is None):
def_list = list_names[0]
else:
def_list = keyValue
sellist = swum.get_List(def_list, USER_CREATED)
dsstr = "["
for i in range(len(sellist)):
dsstr = dsstr + str(sellist[i])
if (i == (len(sellist) - 1)):
dsstr = dsstr + "]"
else:
dsstr = dsstr + ","
else:
list_names = ["No User lists defined"]
def_list = "No User lists defined"
sellist = "User defined list"
listssel = {
"default": def_list,
"list": list_names,
"callback": "select_list"
}
selectDicts.append(listssel)
from dfcleanser.common.common_utils import get_select_defaults
get_select_defaults(list_maint_input_form, maint_list_utility_input_id,
maint_list_utility_input_idList,
maint_list_utility_input_typeList, selectDicts)
list_maint_input_form.set_gridwidth(700)
if (loadfile is None):
list_maint_input_form.set_buttonstyle({
"font-size": 13,
"height": 75,
"width": 90,
"left-margin": 20
})
else:
list_maint_input_form.set_buttonstyle({
"font-size": 13,
"height": 75,
"width": 90,
"left-margin": 205
})
list_maint_input_form.set_fullparms(True)
cfg.drop_config_value(maint_list_utility_input_id + "Parms")
cfg.drop_config_value(maint_list_utility_input_id + "ParmsProtect")
if (not (loadfile is None)):
import json
from dfcleanser.common.common_utils import does_file_exist
if (does_file_exist(loadfile)):
try:
with open(loadfile, 'r') as ds_file:
ds = json.load(ds_file)
ds_file.close()
dsstr = "["
for i in range(len(ds)):
dsstr = dsstr + str(ds[i])
if (i == (len(ds) - 1)):
dsstr = dsstr + "]"
else:
dsstr = dsstr + ","
except Exception as e:
opstat.set_status(False)
opstat.set_errorMsg("Error processing user file to load" +
loadfile)
opstat.set_exception(e)
else:
opstat.set_status(False)
opstat.set_errorMsg("invalid user file to load" + loadfile)
if (opstat.get_status()):
if (loadfile is None):
cfg.set_config_value(maint_list_utility_input_id + "Parms",
[def_list, "", dsstr, ""])
else:
cfg.set_config_value(maint_list_utility_input_id + "Parms",
[def_list, "", dsstr, loadfile])
cfg.set_config_value(maint_list_utility_input_id + "ParmsProtect",
[True, False, True, True])
help_note = "To add a user list enter parms and values above and click on 'Add User List'.</br>To update the current list change values and click on 'Update User List'"
from dfcleanser.common.common_utils import get_help_note_html
list_maint_notes_html = get_help_note_html(help_note, 80, 75, None)
list_maint_html = ""
list_maint_html = list_maint_input_form.get_html()
list_maint_title_html = "<div>User Lists</div><br></br>"
gridclasses = [
"dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"
]
gridhtmls = [
list_maint_title_html, list_maint_html, list_maint_notes_html
]
#print(list_maint_html)
#print(list_maint_notes_html)
print("\n")
display_generic_grid("sw-utils-listdict-wrapper", gridclasses,
gridhtmls)
else:
display_exception(opstat)
add_error_to_log("[Get User Dict from File] " + loadfile +
str(sys.exc_info()[0].__name__))
def display_dict_maint(keyValue=None, loadfile=None):
"""
* ------------------------------------------------------------------------
* function : display the user dicts maintenance form
*
* parms :
* keyValue - dict name
*
* -------------------------------------------------------------------------
"""
#print("display_dict_maint",keyValue,loadfile)
opstat = opStatus()
if (loadfile is None):
dict_maint_input_form = InputForm(
maint_dict_utility_input_id, maint_dict_utility_input_idList,
maint_dict_utility_input_labelList,
maint_dict_utility_input_typeList,
maint_dict_utility_input_placeholderList,
maint_dict_utility_input_jsList, maint_dict_utility_input_reqList)
else:
dict_maint_input_form = InputForm(
maint_dict_file_utility_input_id,
maint_dict_file_utility_input_idList,
maint_dict_file_utility_input_labelList,
maint_dict_file_utility_input_typeList,
maint_dict_file_utility_input_placeholderList,
maint_dict_file_utility_input_jsList,
maint_dict_file_utility_input_reqList)
selectDicts = []
from dfcleanser.sw_utilities.sw_utility_model import get_dicts_names, USER_CREATED
dict_names = get_dicts_names(USER_CREATED)
#print("dict_names",dict_names)
if (not (dict_names is None)):
if (keyValue is None):
def_dict = dict_names[0]
else:
def_dict = keyValue
seldict = swum.get_Dict(def_dict, USER_CREATED)
keys = list(seldict.keys())
if ((def_dict == "Country_Codes") or (def_dict == "Language_Codes")):
keys.sort()
seldict = swum.get_pretty_dict(seldict, keys)
else:
dict_names = ["No User dicts defined"]
def_dict = "No User dicts defined"
seldict = "User defined dict"
dictssel = {
"default": def_dict,
"list": dict_names,
"callback": "select_dict"
}
selectDicts.append(dictssel)
from dfcleanser.common.common_utils import get_select_defaults
get_select_defaults(dict_maint_input_form, maint_dict_utility_input_id,
maint_dict_utility_input_idList,
maint_dict_utility_input_typeList, selectDicts)
dict_maint_input_form.set_gridwidth(700)
#dict_maint_input_form.set_custombwidth(110)
if (loadfile is None):
dict_maint_input_form.set_buttonstyle({
"font-size": 13,
"height": 75,
"width": 90,
"left-margin": 20
})
else:
dict_maint_input_form.set_buttonstyle({
"font-size": 13,
"height": 75,
"width": 90,
"left-margin": 205
})
dict_maint_input_form.set_fullparms(True)
cfg.drop_config_value(maint_dict_utility_input_id + "Parms")
cfg.drop_config_value(maint_dict_utility_input_id + "ParmsProtect")
if (not (loadfile is None)):
import json
#from dfcleanser.common.common_utils import does_file_exist
#print("does_file_exist",does_file_exist(loadfile))
try:
with open(loadfile, 'r') as ds_file:
ds = json.load(ds_file)
ds_file.close()
keys = list(ds.keys())
seldict = swum.get_pretty_dict(ds, keys)
#print(seldict)
except Exception as e:
opstat.set_errorMsg("invalid user file to load " + loadfile)
opstat.set_exception(e)
if (opstat.get_status()):
if (loadfile is None):
cfg.set_config_value(maint_dict_utility_input_id + "Parms",
[def_dict, "", seldict, ""])
else:
cfg.set_config_value(maint_dict_utility_input_id + "Parms",
[def_dict, "", seldict, loadfile])
cfg.set_config_value(maint_dict_utility_input_id + "ParmsProtect",
[True, False, True, True])
help_note = "To add a user dict enter parms and values above and click on 'Add User Dict'.</br>To update the current dict change values and click on 'Update User Dict'"
from dfcleanser.common.common_utils import get_help_note_html
dict_maint_notes_html = get_help_note_html(help_note, 80, 75, None)
dict_maint_html = "Fill in new user dict parms or update currently displayed user dict."
dict_maint_html = dict_maint_input_form.get_html()
dict_maint_title_html = "<div>User Dicts</div><br></br>"
gridclasses = [
"dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"
]
gridhtmls = [
dict_maint_title_html, dict_maint_html, dict_maint_notes_html
]
#print(dict_maint_html)
#print(dict_maint_notes_html)
print("\n")
display_generic_grid("sw-utils-listdict-wrapper", gridclasses,
gridhtmls)
else:
display_exception(opstat)
add_error_to_log("[Get User Dict from File] " + loadfile +
str(sys.exc_info()[0].__name__))
def display_inspect_cols(parms):
"""
* --------------------------------------------------------------------------
* function : display the inspect cols option
*
* parms :
*
* returns :
* N/A
* --------------------------------------------------------
"""
opstat = opStatus()
clock = RunningClock()
clock.start()
try:
df = cfg.get_current_chapter_df(cfg.DataInspection_ID)
colnames = df.columns.tolist()
if (not (parms is None)):
colname = parms
else:
colname = colnames[0]
cnames = {
'default': colname,
'list': colnames,
"callback": "change_inspect_cols_col",
"size": 10
}
if (is_numeric_col(df, colname)):
coldetails_form = InputForm(diw.inspect_col_input_id,
diw.inspect_col_input_idList,
diw.inspect_col_input_labelList,
diw.inspect_col_input_typeList,
diw.inspect_col_input_placeholderList,
diw.inspect_col_input_jsList,
diw.inspect_col_input_reqList)
else:
coldetails_form = InputForm(
diw.inspect_nn_col_input_id, diw.inspect_nn_col_input_idList,
diw.inspect_nn_col_input_labelList,
diw.inspect_nn_col_input_typeList,
diw.inspect_nn_col_input_placeholderList,
diw.inspect_nn_col_input_jsList,
diw.inspect_nn_col_input_reqList)
selectDicts = []
selectDicts.append(cnames)
get_select_defaults(coldetails_form, diw.inspect_col_input_id,
diw.inspect_col_input_idList,
diw.inspect_col_input_typeList, selectDicts)
coldetails_form.set_shortForm(True)
coldetails_form.set_fullparms(True)
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
coldetails_form.set_gridwidth(360)
if (is_numeric_col(df, colname)):
coldetails_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 85,
"left-margin": 2
})
else:
coldetails_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 85,
"left-margin": 75
})
else:
coldetails_form.set_gridwidth(480)
if (is_numeric_col(df, colname)):
coldetails_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 110,
"left-margin": 2
})
else:
coldetails_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 110,
"left-margin": 110
})
coldetails_html = coldetails_form.get_html()
from dfcleanser.data_cleansing.data_cleansing_widgets import display_col_stats
col_stats_html = display_col_stats(df, colname, False, True)
gridclasses = ["dfc-left", "dfc-right"]
gridhtmls = [col_stats_html, coldetails_html]
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
display_generic_grid("df-inspection-column-data-wrapper",
gridclasses, gridhtmls)
else:
display_generic_grid("df-inspection-pop-up-column-data-wrapper",
gridclasses, gridhtmls)
except Exception as e:
opstat.store_exception("Error displaying column data\n ", e)
clock.stop()
if (not (opstat.get_status())):
display_exception(opstat)
def display_dc_data_scripting(optionId, parms=None):
from IPython.display import clear_output
clear_output()
from dfcleanser.common.cfg import check_if_dc_init
if (not check_if_dc_init()):
from dfcleanser.common.display_utils import display_dfcleanser_taskbar
display_dfcleanser_taskbar(
ButtonGroupForm(dc_script_tb_id, dc_script_tb_keyTitleList,
dc_script_tb_jsList, dc_script_tb_centered), False)
from dfcleanser.scripting.data_scripting_control import clear_data_scripting_data
clear_data_scripting_data()
return
from dfcleanser.common.display_utils import display_dfcleanser_taskbar
display_dfcleanser_taskbar(
ButtonGroupForm(dc_script_tb_id, dc_script_tb_keyTitleList,
dc_script_tb_jsList, dc_script_tb_centered), False)
if (parms == None):
from dfcleanser.scripting.data_scripting_control import clear_data_scripting_data
clear_data_scripting_data()
else:
funcid = int(parms[0])
if (funcid == dsm.TURN_ON_SCRIPTING):
from dfcleanser.scripting.data_scripting_control import set_scripting_status
set_scripting_status(True)
from dfcleanser.common.common_utils import display_status_note
display_status_note("Scripting is turned on")
elif (funcid == dsm.TURN_OFF_SCRIPTING):
from dfcleanser.scripting.data_scripting_control import set_scripting_status
set_scripting_status(False)
from dfcleanser.common.common_utils import display_status_note
display_status_note("Scripting is turned off")
elif (funcid == dsm.SHOW_CURRENT_SCRIPT):
display_scripting_forms()
elif (funcid == dsm.ADD_TO_CURRENT_SCRIPT):
script_form = InputForm(
dc_add_code_input_id, dc_add_code_input_idList,
dc_add_code_input_labelList, dc_add_code_input_typeList,
dc_add_code_input_placeholderList, dc_add_code_input_jsList,
dc_add_code_input_reqList)
script_form.set_shortForm(True)
script_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 140,
"left-margin": 70
})
script_form.set_gridwidth(880)
script_form.set_fullparms(True)
script_form_html = script_form.get_html()
script_title_html = "<div>Scripting</div><br>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"]
gridhtmls = [script_title_html, script_form_html]
from dfcleanser.common.common_utils import display_generic_grid
display_generic_grid("data-scripting-wrapper", gridclasses,
gridhtmls)
elif (funcid == dsm.ADD_CODE_SCRIPT):
from dfcleanser.scripting.data_scripting_control import add_code_to_script
add_code_to_script(parms)
display_status("Code added to Current Script succesfully ")
elif (funcid == dsm.DELETE_CURRENT_SCRIPT):
from dfcleanser.scripting.data_scripting_control import drop_current_script
drop_current_script()
display_status("Current Script Cleared succesfully ")
elif (funcid == dsm.LOAD_BACKUP_SCRIPT):
from dfcleanser.scripting.data_scripting_control import load_backup_scriptlog_to_current
load_backup_scriptlog_to_current()
display_scripting_forms()
display_status("Current Script Loaded from Backup ")
elif (funcid == dsm.SAVE_BACKUP_SCRIPT):
codeparms = get_parms_for_input(parms[1], dc_script_input_idList)
save_code = get_code_from_form(codeparms)
from dfcleanser.scripting.data_scripting_control import (
set_current_scriptlog, save_current_scriptlog_to_backup)
set_current_scriptlog(save_code)
save_current_scriptlog_to_backup(save_code)
display_scripting_forms()
display_status("Current Script Backed up successfully ")
elif (funcid == dsm.RUN_CURRENT_SCRIPT):
opstat = opStatus()
from dfcleanser.scripting.data_scripting_control import run_scriptlog
run_scriptlog(parms, opstat)
if (opstat.get_status()):
display_status("Current Script Run successfully ")
else:
display_exception(opstat)
