def display_df_dataframes(title):
"""
* --------------------------------------------------------------------------
* function : display dataframe manager form
*
* parms :
* N/A
*
* returns : N/A
* --------------------------------------------------------
"""
dfmgr_input_html = get_df_list_html(title)
dfmgr_heading_html = "<div>Current dfCleanser dataframes </div>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-main"]
gridhtmls = [dfmgr_heading_html, dfmgr_input_html]
print("\n")
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses,
gridhtmls)
else:
display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses,
gridhtmls, True)
from dfcleanser.common.display_utils import display_pop_up_buffer
display_pop_up_buffer()
def display_system_main_taskbar():
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
from dfcleanser.common.display_utils import display_dfcleanser_taskbar
display_dfcleanser_taskbar(
ButtonGroupForm(system_environment_doc_id,
system_environment_keyTitleList,
system_environment_jsList,
system_environment_centered))
else:
system_tb_A = ButtonGroupForm(system_environmentA_doc_id,
system_environmentA_keyTitleList,
system_environmentA_jsList,
system_environmentA_centered)
system_tb_A.set_gridwidth(480)
system_tb_A_html = system_tb_A.get_html()
system_tb_B = ButtonGroupForm(system_environmentB_doc_id,
system_environmentB_keyTitleList,
system_environmentB_jsList,
system_environmentB_centered)
system_tb_B.set_gridwidth(480)
system_tb_B_html = system_tb_B.get_html()
gridclasses = ["dfc-top-", "dfc-footer"]
gridhtmls = [system_tb_A_html, system_tb_B_html]
display_generic_grid("dfcleanser-system-tb-pop-up-wrapper",
gridclasses, gridhtmls)
def display_get_zipcode_attributes(parms):
"""
* ---------------------------------------------------------
* function : display the calculate distance form
*
* parms :
* pytype - address or coords
*
* returns :
* N/A
* --------------------------------------------------------
"""
zip_attr_form = InputForm(
zipcode_atributes_input_id, zipcode_atributes_input_idList,
zipcode_atributes_input_labelList, zipcode_atributes_input_typeList,
zipcode_atributes_input_placeholderList,
zipcode_atributes_input_jsList, zipcode_atributes_input_reqList)
zip_attr_form.set_gridwidth(720)
zip_attr_form.set_custombwidth(100)
zip_attr_form.set_fullparms(True)
zip_attr_input_html = ""
zip_attr_input_html = zip_attr_form.get_html()
zip_attr_heading_html = "<div>Get Zipcode Attributes</div>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"]
gridhtmls = [zip_attr_heading_html, zip_attr_input_html]
print("\n")
display_generic_grid("geocode-utility-wrapper", gridclasses, gridhtmls)
def display_inspect_outliers(colname):
"""
* --------------------------------------------------------------------------
* function : display the inspect outliers option
*
* parms :
*
* returns :
* N/A
* --------------------------------------------------------
"""
opstat = opStatus()
display_inspect_cols(colname)
outliers_html = diw.get_simple_outliers(cfg.get_current_chapter_df(
cfg.DataInspection_ID),
colname,
opstat,
display=False)
gridclasses = ["dfc-main"]
gridhtmls = [outliers_html]
print("\n")
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
display_generic_grid("df-inspection-outliers-data-wrapper",
gridclasses, gridhtmls)
else:
display_generic_grid("df-inspection-outliers-pop-up-data-wrapper",
gridclasses, gridhtmls)
def display_df_criteria(df_title, df):
"""
* --------------------------------------------------------------------------
* function : display current df subset form
*
* parms :
* df - dataframe to subset from
* filters - filters form
* colname - filters column name
*
* returns : N/A
* --------------------------------------------------------
"""
col_stats_table = get_column_stats_table(df_title, df)
from dfcleanser.common.html_widgets import InputForm
subset_input_form = InputForm(get_subset_criteria_input_id,
get_subset_criteria_input_idList,
get_subset_criteria_input_labelList,
get_subset_criteria_input_typeList,
get_subset_criteria_input_placeholderList,
get_subset_criteria_input_jsList,
get_subset_criteria_input_reqList)
subset_input_form.set_label_note({
2: [
dfchelp.SUBSET_CRITERIA, "dataframe subset criteria",
"subsetcriteriaimageid"
]
})
subset_input_form.set_textarea_resize_flag(1, False)
subset_input_form.set_textarea_resize_flag(3, False)
subset_input_form.set_shortForm(False)
subset_input_form.set_gridwidth(620)
subset_input_form.set_custombwidth(110)
subset_input_form.set_fullparms(True)
subset_input_form.set_custom_font_size("gsselectstring", 14)
cfg.set_config_value(get_subset_criteria_input_id + "Parms", [
"", swsm.starting_criteria_preamble, swsm.starting_criteria,
swsm.starting_criteria_postamble
])
cfg.set_config_value(get_subset_criteria_input_id + "ParmsProtect",
[False, True, False, True])
get_subset_input_html = subset_input_form.get_html()
get_subset_heading_html = "<div>Get Dataframe Subset</div><br></br>"
gridclasses = ["dfc-top", "dfcleanser-common-grid-header", "dfc-bottom"]
gridhtmls = [
col_stats_table, get_subset_heading_html, get_subset_input_html
]
print("\n")
display_generic_grid("sw-utils-subset-wrapper", gridclasses, gridhtmls)
def display_data_export_parms(title,
plist,
fparms,
exportID,
fname,
custom=False,
dbnote=False):
"""
* --------------------------------------------------------------------------
* function : display export parms
*
* parms :
* title - poarms title
* plist - label list
* fparms - values list
* exportID - export table id
* fname - file name
* dbnote - dbnote flag
*
* returns : N/A
* --------------------------------------------------------
"""
#print("display_data_export_parms",title,plist,fparms)
parms_html = displayParms(title, plist, fparms, exportID, 100, 2, False)
if (custom):
status_html = display_status(
"Custom export code Exported successfully", False, False)
exportnotes = [
"[Total Export Time] : " +
str(get_formatted_time(time.time() - get_data_export_start())) +
" seconds",
"( check if df exists via dfcleanser.common.cfg.is_a_dfc_dataframe_loaded() )"
]
else:
if (dbnote):
status_html = display_status(
" Dataframe Exported successfully to table " + fname, False,
False)
else:
status_html = display_status(
" Dataframe Exported successfully to File " + fname, False,
False)
exportnotes = [
"[Total Export Time] : " +
str(get_formatted_time(time.time() - get_data_export_start())) +
" seconds"
]
notes_html = display_notes(exportnotes, False)
gridclasses = ["dfc-top", "dfc-left", "dfc-right"]
gridhtmls = [parms_html, status_html, notes_html]
display_generic_grid("data-import-stats-wrapper", gridclasses, gridhtmls)
def display_saved_subset_sequences():
"""
* --------------------------------------------------------------------------
* function : display current saved sequences
*
* parms :
* df - dataframe to subset from
*
* returns : N/A
* --------------------------------------------------------
"""
from dfcleanser.common.html_widgets import InputForm
subset_input_form = InputForm(get_subset_sequences_input_id,
get_subset_sequences_input_idList,
get_subset_sequences_input_labelList,
get_subset_sequences_input_typeList,
get_subset_sequences_input_placeholderList,
get_subset_sequences_input_jsList,
get_subset_sequences_input_reqList)
selectDicts = []
sequences = swsm.get_sequences_list()
if (len(sequences) > 0):
seqdict = {"default": sequences[0], "list": sequences}
else:
seqdict = {
"default": "no saved sequences",
"list": ["no saved sequences"]
}
selectDicts.append(seqdict)
subsetact = {"default": "Auto Run", "list": ["Auto Run", "Manual Run"]}
selectDicts.append(subsetact)
get_select_defaults(subset_input_form, get_subset_sequences_input_form[0],
get_subset_sequences_input_form[1],
get_subset_sequences_input_form[3], selectDicts)
subset_input_form.set_shortForm(False)
subset_input_form.set_gridwidth(680)
subset_input_form.set_custombwidth(140)
subset_input_form.set_fullparms(True)
get_subset_input_html = subset_input_form.get_html()
get_subset_heading_html = "<div>Run Saved Subset Sequence</div><br>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-main"]
gridhtmls = [get_subset_heading_html, get_subset_input_html]
print("\n")
display_generic_grid("sw-utils-subset-main-wrapper", gridclasses,
gridhtmls)
def display_sequence_save_subset(df_title, df, auto=False):
"""
* --------------------------------------------------------------------------
* function : display current df subset run
*
* parms :
* df - dataframe to subset from
*
* returns : N/A
* --------------------------------------------------------
"""
df_stats_table = get_df_stats_table(df_title, df)
df_cols_table = get_column_stats_table(df_title, df)
from dfcleanser.common.html_widgets import InputForm
subset_input_form = InputForm(
get_saved_save_input_id, get_saved_save_input_idList,
get_saved_save_input_labelList, get_saved_save_input_typeList,
get_saved_save_input_placeholderList, get_saved_save_input_jsList,
get_saved_save_input_reqList)
selectDicts = []
dropsel = {"default": "False", "list": ["True", "False"]}
selectDicts.append(dropsel)
get_select_defaults(subset_input_form, get_saved_save_input_id,
get_saved_save_input_idList,
get_saved_save_input_typeList, selectDicts)
subset_input_form.set_shortForm(False)
subset_input_form.set_gridwidth(680)
subset_input_form.set_custombwidth(140)
subset_input_form.set_fullparms(True)
cfg.set_config_value(
get_saved_save_input_id + "Parms",
[df_title, "./" + df_title + ".csv", df_title, "False"])
cfg.drop_config_value(get_saved_save_input_id + "ParmsProtect")
get_subset_input_html = subset_input_form.get_html()
get_subset_heading_html = "<div>Save Dataframe Subset</div><br>"
gridclasses = [
"dfc-top", "dfc-main", "dfcleanser-common-grid-header", "dfc-bottom"
]
gridhtmls = [
df_stats_table, df_cols_table, get_subset_heading_html,
get_subset_input_html
]
print("\n")
display_generic_grid("sw-utils-subset-run-wrapper", gridclasses, gridhtmls)
def display_offline():
"""
* --------------------------------------------------------------------------
* function : display offline toggle mode
*
* parms :
* N/A
*
* returns : N/A
* --------------------------------------------------------
"""
from dfcleanser.common.html_widgets import InputForm
dfmanager_input_form = InputForm(
dfmgr_offline_input_id, dfmgr_offline_input_idList,
dfmgr_offline_input_labelList, dfmgr_offline_input_typeList,
dfmgr_offline_input_placeholderList, dfmgr_offline_input_jsList,
dfmgr_offline_input_reqList)
selectDicts = []
if (sysm.get_dfc_run_offline_status()):
stat = "True"
else:
stat = "False"
offline_mode = {"default": stat, "list": ["False", "True"]}
selectDicts.append(offline_mode)
get_select_defaults(dfmanager_input_form, dfmgr_offline_input_id,
dfmgr_offline_input_idList,
dfmgr_offline_input_typeList, selectDicts)
dfmanager_input_form.set_shortForm(True)
dfmanager_input_form.set_gridwidth(480)
dfmanager_input_form.set_custombwidth(100)
dfmanager_input_form.set_fullparms(True)
dfmgr_input_html = dfmanager_input_form.get_html()
dfmgr_heading_html = "<div>Run dfc Offline Manager</div>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-main"]
gridhtmls = [dfmgr_heading_html, dfmgr_input_html]
print("\n")
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses,
gridhtmls)
else:
display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses,
gridhtmls, True)
from dfcleanser.common.display_utils import display_pop_up_buffer
display_pop_up_buffer()
def process_county_cities(parms):
opstat = opStatus()
fparms = get_parms_for_input(parms, suzw.county_cities_input_idList)
state = fparms[0][:2]
county = fparms[1]
cfg.set_config_value(suzw.county_cities_input_id + "Parms", fparms)
suzw.display_get_cities_for_county(parms)
print("\n")
citiesHeader = [""]
citiesRows = []
citiesWidths = [20, 80]
citiesAligns = ["left", "left"]
primary_cities = suzm.get_cities_for_county(state,
county,
city_type=suzm.ANY_CITY_TYPE)
if (not (primary_cities is None)):
citiesRows.append(["US Zipcode Cities", str(primary_cities)])
cities_table = None
from dfcleanser.common.table_widgets import dcTable, get_row_major_table, ROW_MAJOR, SCROLL_DOWN
cities_table = dcTable(
"Cities For " + str(county) + " - " +
str(suzm.get_state_name(state).upper()), 'citiescodesid',
cfg.SWZipcodeUtility_ID, citiesHeader, citiesRows, citiesWidths,
citiesAligns)
cities_table.set_small(True)
cities_table.set_checkLength(False)
cities_table.set_border(True)
cities_table.set_tabletype(ROW_MAJOR)
cities_table.set_rowspertable(50)
citiesHtml = get_row_major_table(cities_table, SCROLL_DOWN, False)
gridclasses = ["dfc-top"]
gridhtmls = [citiesHtml]
display_generic_grid("display-geocode-coords-wrapper", gridclasses,
gridhtmls)
def display_get_cities_for_state(parms):
state_cities_form = InputForm(
state_cities_input_id, state_cities_input_idList,
state_cities_input_labelList, state_cities_input_typeList,
state_cities_input_placeholderList, state_cities_input_jsList,
state_cities_input_reqList)
selectDicts = []
from dfcleanser.sw_utilities.sw_utility_model import get_Dict
states_dict = get_Dict("US_States_and_Territories")
state_keys = list(states_dict.keys())
state_keys.sort()
states_list = []
for i in range(len(state_keys)):
states_list.append(
str(state_keys[i]) + " : " + str(states_dict.get(state_keys[i])))
state_sel = {"default": states_list[0], "list": states_list}
selectDicts.append(state_sel)
get_select_defaults(state_cities_form, state_cities_input_id,
state_cities_input_idList, state_cities_input_typeList,
selectDicts)
state_cities_form.set_gridwidth(720)
state_cities_form.set_custombwidth(100)
state_cities_form.set_fullparms(True)
state_cities_input_html = ""
state_cities_input_html = state_cities_form.get_html()
state_cities_heading_html = "<div>Get State Cities</div>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"]
gridhtmls = [state_cities_heading_html, state_cities_input_html]
print("\n")
display_generic_grid("geocode-utility-wrapper", gridclasses, gridhtmls)
def display_scripting_forms():
from dfcleanser.scripting.data_scripting_control import get_current_scriptlog
if (not (get_current_scriptlog() == None)):
cfg.set_config_value(dc_script_input_id + "Parms",
[get_current_scriptlog()])
print("\n")
notes = []
notes.append("You can manually edit the script log as desired")
notes.append(
"The step numbers are cmmentary only and order is insignificant")
display_notes(notes)
script_form = InputForm(dc_script_input_id, dc_script_input_idList,
dc_script_input_labelList,
dc_script_input_typeList,
dc_script_input_placeholderList,
dc_script_input_jsList, dc_script_input_reqList)
script_form.set_shortForm(True)
script_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 140,
"left-margin": 70
})
script_form.set_gridwidth(880)
script_form.set_fullparms(True)
script_form_html = script_form.get_html()
script_title_html = "<div>Scripting</div><br>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"]
gridhtmls = [script_title_html, script_form_html]
from dfcleanser.common.common_utils import display_generic_grid
display_generic_grid("data-scripting-wrapper", gridclasses, gridhtmls)
cfg.drop_config_value(dc_script_input_id + "Parms")
print("\n")
def display_add_df_input():
"""
* --------------------------------------------------------------------------
* function : add a dataframe to the dfc list
*
* parms :
*
* returns : N/A
* --------------------------------------------------------
"""
from dfcleanser.common.html_widgets import InputForm
dfmanager_input_form = InputForm(
dfmgr_add_input_id, dfmgr_add_input_idList, dfmgr_add_input_labelList,
dfmgr_add_input_typeList, dfmgr_add_input_placeholderList,
dfmgr_add_input_jsList, dfmgr_add_input_reqList)
dfmanager_input_form.set_shortForm(True)
dfmanager_input_form.set_gridwidth(480)
dfmanager_input_form.set_custombwidth(70)
dfmanager_input_form.set_fullparms(True)
dfmgr_input_html = dfmanager_input_form.get_html()
dfmgr_heading_html = "<div>Add df to dfc dataframe Manager</div>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-main"]
gridhtmls = [dfmgr_heading_html, dfmgr_input_html]
print("\n")
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses,
gridhtmls)
else:
display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses,
gridhtmls, True)
from dfcleanser.common.display_utils import display_pop_up_buffer
display_pop_up_buffer()
def show_sys_info():
"""
* --------------------------------------------------------------------------
* function : display dfcleanser system info
*
* parms :
* N/A
*
* returns : N/A
* --------------------------------------------------------
"""
title = "Installed Python Info"
ptitles = ["Version", "API", "Info"]
pvalues = [
str(get_python_version()),
str(sys.api_version),
str(sys.version_info)
]
parms_html = displayParms(title, ptitles, pvalues, cfg.System_ID, None, 0,
False, 11)
notes_html = show_setup_notes()
libs_html = show_libs_info()
libs_notes_html = show_libs_notes()
gridclasses = ["dfc-header", "dfc-top", "dfc-bottom", "dfc-footer"]
gridhtmls = [parms_html, notes_html, libs_html, libs_notes_html]
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
display_generic_grid("df-sys-info-wrapper", gridclasses, gridhtmls)
else:
display_generic_grid("df-sys-info-pop-up-wrapper", gridclasses,
gridhtmls)
from dfcleanser.common.display_utils import display_pop_up_buffer
display_pop_up_buffer()
def display_system_chapters_taskbar():
"""
* --------------------------------------------------------------------------
* function : display system taskbar
*
* parms :
* N/A
*
* returns : N/A
* --------------------------------------------------------
"""
print("\n")
dfc_utils_modules_checkbox = CheckboxGroupForm(
dfc_utils_modules_checkbox_id, dfc_utils_modules_checkbox_idList,
dfc_utils_modules_checkbox_labelList,
dfc_utils_modules_checkbox_jsList,
get_current_checkboxes(sysm.UTILITIES))
if (not (cfg.get_dfc_mode() == cfg.INLINE_MODE)):
dfc_utils_modules_checkbox.set_rows_count([2, 2, 1])
dfc_utils_modules_heading_html = "<div>dfcleanser Utilities</div><br>"
dfc_utils_modules_checkbox_html = dfc_utils_modules_checkbox.get_html()
gridclasses = ["dfcleanser-common-grid-header", "dfc-main"]
gridhtmls = [
dfc_utils_modules_heading_html, dfc_utils_modules_checkbox_html
]
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
display_generic_grid("dfcleanser-chapters-wrapper", gridclasses,
gridhtmls)
else:
display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses,
gridhtmls, True)
dfc_script_modules_tb = ButtonGroupForm(system_chapters_tb_doc_id,
system_chapters_tb_keyTitleList,
system_chapters_tb_jsList,
system_chapters_tb_centered)
dfc_script_modules_tb.set_gridwidth(480)
dfc_script_modules_tb.set_custombwidth(160)
dfc_script_modules_tb_html = dfc_script_modules_tb.get_html()
gridclasses = ["dfc-footer"]
gridhtmls = [dfc_script_modules_tb_html]
display_generic_grid("dfcleanser-chapters-tb-wrapper", gridclasses,
gridhtmls)
print("\n")
def display_inspect_datatypes(option, df_data_info):
"""
* --------------------------------------------------------------------------
* function : display the datatypes option
*
* parms :
*
* returns :
* N/A
* --------------------------------------------------------
"""
opstat = opStatus()
import matplotlib.pyplot as plt
clock = RunningClock()
clock.start()
try:
if (not (option == dim.DISPLAY_FULL_COLUMN_NAMES)):
data_types_table = dcTable("Column Data Types", "datatypesTable",
cfg.DataInspection_ID)
else:
data_types_table = None
data_types_html = diw.display_df_datatypes(data_types_table,
df_data_info[0],
df_data_info[1],
df_data_info[2], option,
False)
gridclasses = ["dfc-main"]
gridhtmls = [data_types_html]
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
display_generic_grid("df-inspection-wrapper", gridclasses,
gridhtmls)
else:
display_generic_grid("df-inspection-pop-up-wrapper", gridclasses,
gridhtmls)
print("\n")
import matplotlib.pyplot as plt
import numpy as np
font = {'fontsize': 14}
font2 = {'fontsize': 18}
objects = []
for i in range(len(df_data_info[0])):
ttype = str(df_data_info[0][i])
ttype = ttype.replace("datetime.", "")
objects.append(ttype)
y_pos = np.arange(len(objects))
plt.bar(y_pos,
df_data_info[1],
align='center',
alpha=0.5,
color='#428bca')
plt.xticks(y_pos, objects, rotation='vertical')
plt.ylabel('Type Counts', fontdict=font)
plt.xlabel('Data Types', fontdict=font)
plt.title('Column Data Types', fontdict=font2)
plt.show()
except Exception as e:
opstat.store_exception("Error displaying data types\n ", e)
clock.stop()
if (not (opstat.get_status())):
display_exception(opstat)
def display_inspect_rows(rowid=0):
"""
* --------------------------------------------------------------------------
* function : display the inspect rows option
*
* parms :
*
* returns :
* N/A
* --------------------------------------------------------
"""
opstat = opStatus()
clock = RunningClock()
clock.start()
try:
print("\n")
from dfcleanser.data_transform.data_transform_dataframe_widgets import display_current_df_index
display_current_df_index(
cfg.get_current_chapter_df(cfg.DataInspection_ID),
cfg.get_current_chapter_dfc_df_title(cfg.DataInspection_ID), 0,
True)
row_stats_html = diw.display_row_stats(
cfg.get_current_chapter_df(cfg.DataInspection_ID),
cfg.get_config_value(cfg.CURRENT_INSPECTION_DF), False)
sample_row_html = dim.display_df_rows(
cfg.get_current_chapter_df(cfg.DataInspection_ID), rowid, 200)
rows_openexcel_tb = diw.get_inspection_openexcel_taskbar()
rows_openexcel_tb.set_gridwidth(620)
rows_openexcel_tb.set_customstyle({
"font-size": 13,
"height": 90,
"width": 120,
"left-margin": 10
})
rows_openexcel_html = rows_openexcel_tb.get_html()
rows_openexcel_html = (rows_openexcel_html + "<br>")
cfg.set_config_value(cfg.CURRENT_SCROLL_ROW_KEY, rowid)
gridclasses = ["dfc-top", "dfc-bottom", "dfc-footer"]
gridhtmls = [row_stats_html, sample_row_html, rows_openexcel_html]
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
display_generic_grid("df-inspection-row-data-wrapper", gridclasses,
gridhtmls)
else:
display_generic_grid("df-inspection-row-data-pop-up-wrapper",
gridclasses, gridhtmls)
except Exception as e:
opstat.store_exception("Error displaying row data\n ", e)
display_exception(opstat)
import traceback
traceback.print_exc()
clock.stop()
def display_get_cities_for_county(parms):
if (parms is None):
fparms = cfg.get_config_value(county_cities_input_id + "Parms")
if (not (fparms is None)):
stateid = fparms[0]
else:
stateid = None
else:
fparms = get_parms_for_input(parms, county_cities_input_idList)
stateid = fparms[0]
cfg.set_config_value(county_cities_input_id + "Parms", fparms)
county_cities_form = InputForm(
county_cities_input_id, county_cities_input_idList,
county_cities_input_labelList, county_cities_input_typeList,
county_cities_input_placeholderList, county_cities_input_jsList,
county_cities_input_reqList)
selectDicts = []
from dfcleanser.sw_utilities.sw_utility_model import get_Dict
states_dict = get_Dict("US_States_and_Territories")
state_keys = list(states_dict.keys())
state_keys.sort()
states_list = []
for i in range(len(state_keys)):
states_list.append(
str(state_keys[i]) + " : " + str(states_dict.get(state_keys[i])))
if (stateid is None):
state_def = states_list[0]
else:
state_def = stateid
state_sel = {
"default": state_def,
"list": states_list,
"callback": "change_state_for_counties"
}
selectDicts.append(state_sel)
state_id = state_def[:2]
counties_list = suzm.get_counties_for_state(state_id)
county_sel = {"default": counties_list[0], "list": counties_list}
selectDicts.append(county_sel)
get_select_defaults(county_cities_form, county_cities_input_id,
county_cities_input_idList,
county_cities_input_typeList, selectDicts)
county_cities_form.set_gridwidth(720)
county_cities_form.set_custombwidth(100)
county_cities_form.set_fullparms(True)
county_cities_input_html = ""
county_cities_input_html = county_cities_form.get_html()
county_cities_heading_html = "<div>Get County Cities</div>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"]
gridhtmls = [county_cities_heading_html, county_cities_input_html]
print("\n")
display_generic_grid("geocode-utility-wrapper", gridclasses, gridhtmls)
def display_df_subset_setup():
"""
* --------------------------------------------------------------------------
* function : display current df subset form
*
* parms :
* df - dataframe to subset from
* filters - filters form
* colname - filters column name
*
* returns : N/A
* --------------------------------------------------------
"""
df_title = cfg.get_config_value(cfg.CURRENT_SUBSET_DF)
df = cfg.get_dfc_dataframe_df(df_title)
col_stats_table = get_column_stats_table(df_title, df)
from dfcleanser.common.html_widgets import InputForm
subset_input_form = InputForm(get_subset_input_id, get_subset_input_idList,
get_subset_input_labelList,
get_subset_input_typeList,
get_subset_input_placeholderList,
get_subset_input_jsList,
get_subset_input_reqList)
selectDicts = []
dataframes = cfg.get_dfc_dataframes_select_list(cfg.SWDFSubsetUtility_ID)
selectDicts.append(dataframes)
current_df = cfg.get_current_chapter_df(cfg.SWDFSubsetUtility_ID)
colnames = current_df.columns.tolist()
cols_name_list = [" "]
for i in range(len(colnames)):
cols_name_list.append(colnames[i])
cnames = {
"default": cols_name_list[0],
"list": cols_name_list,
"callback": "change_subset_cols"
}
selectDicts.append(cnames)
subssel = {"default": "Keep", "list": ["Keep", "Drop"]}
selectDicts.append(subssel)
get_select_defaults(subset_input_form, get_subset_input_form[0],
get_subset_input_form[1], get_subset_input_form[3],
selectDicts)
subset_input_form.set_shortForm(False)
subset_input_form.set_gridwidth(680)
subset_input_form.set_custombwidth(140)
subset_input_form.set_fullparms(True)
get_subset_input_html = subset_input_form.get_html()
get_subset_heading_html = "<div>Get Dataframe Subset</div><br></br>"
gridclasses = ["dfc-top", "dfcleanser-common-grid-header", "dfc-bottom"]
gridhtmls = [
col_stats_table, get_subset_heading_html, get_subset_input_html
]
print("\n")
display_generic_grid("sw-utils-subset-wrapper", gridclasses, gridhtmls)
def display_list_dict(stype, keyValue=None):
"""
* ------------------------------------------------------------------------
* function : display the dict or list start form
*
* parms :
* id - list or dict
* keyValue - dict or list name
*
* -------------------------------------------------------------------------
"""
#print("display_list_dict",stype,keyValue)
if (stype == swum.DICT_ID):
list_dict_form = build_dict_utility_input_form
else:
list_dict_form = build_list_utility_input_form
list_dict_input_form = InputForm(list_dict_form[0], list_dict_form[1],
list_dict_form[2], list_dict_form[3],
list_dict_form[4], list_dict_form[5],
list_dict_form[6])
selectDicts = []
if (stype == swum.DICT_ID):
from dfcleanser.sw_utilities.sw_utility_model import get_dicts_names, DFC_CREATED
dict_names = get_dicts_names(DFC_CREATED)
if (keyValue is None):
def_dict = dict_names[0]
else:
def_dict = keyValue
#print("def_dict",def_dict,keyValue)
dictssel = {
"default": def_dict,
"list": dict_names,
"callback": "select_dict"
}
seldict = swum.get_Dict(def_dict, DFC_CREATED)
keys = list(seldict.keys())
if ((def_dict == "Country_Codes") or (def_dict == "Language_Codes")):
keys.sort()
seldict = swum.get_pretty_dict(seldict, keys)
else:
from dfcleanser.sw_utilities.sw_utility_model import get_lists_names, DFC_CREATED
list_names = get_lists_names(DFC_CREATED)
if (keyValue is None):
def_list = list_names[0]
else:
def_list = keyValue
dictssel = {
"default": def_list,
"list": list_names,
"callback": "select_list"
}
sellist = str(swum.get_List(def_list, DFC_CREATED))
selectDicts.append(dictssel)
from dfcleanser.common.common_utils import get_select_defaults
get_select_defaults(list_dict_input_form, list_dict_form[0],
list_dict_form[1], list_dict_form[3], selectDicts)
list_dict_input_form.set_gridwidth(700)
list_dict_input_form.set_custombwidth(120)
list_dict_input_form.set_fullparms(True)
if (stype == swum.DICT_ID):
selparms = [def_dict, seldict]
cfg.set_config_value(build_dict_utility_input_id + "Parms", selparms)
cfg.set_config_value(build_dict_utility_input_id + "ParmsProtect",
[False, True])
help_note = "To retrieve a dict in python from dfcleanser.sw_utiliities.sw_utility.model call 'get_Dict(dictname)'.</br>To add a dict in python from dfcleanser.sw_utiliities.sw_utility.model call 'add_Dict(dictname,newdict)'"
else:
selparms = [def_list, sellist]
cfg.set_config_value(build_list_utility_input_id + "Parms", selparms)
cfg.set_config_value(build_list_utility_input_id + "ParmsProtect",
[False, True])
help_note = "To retrieve a list in python from dfcleanser.sw_utiliities.sw_utility.model call 'get_List(listname)'.</br>To add a list in python from dfcleanser.sw_utiliities.sw_utility.model call 'add_List(listtname,newlist)'"
from dfcleanser.common.common_utils import get_help_note_html
listdict_notes_html = get_help_note_html(help_note, 100, None, None)
list_dictcustom_html = ""
list_dictcustom_html = list_dict_input_form.get_html()
if (stype == swum.DICT_ID):
list_title_html = "<div>dfc Dicts</div><br></br>"
cfg.drop_config_value(build_dict_utility_input_id + "Parms")
else:
list_title_html = "<div>dfc Lists</div><br></br>"
cfg.drop_config_value(build_list_utility_input_id + "Parms")
gridclasses = ["dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"]
gridhtmls = [list_title_html, list_dictcustom_html, listdict_notes_html]
print("\n")
display_generic_grid("sw-utils-listdict-wrapper", gridclasses, gridhtmls)
def process_zipcode_attributes(parms):
DEBUG_PROC_ZC_ATTRS = False
opstat = opStatus()
fparms = get_parms_for_input(parms, suzw.zipcode_atributes_input_idList)
zipcode = fparms[0]
cfg.set_config_value(suzw.zipcode_atributes_input_id + "Parms", fparms)
suzw.display_get_zipcode_attributes(parms)
print("\n")
zipattrsHeader = [""]
zipattrsRows = []
zipattrsWidths = [30, 70]
zipattrsAligns = ["left", "left"]
primary_cities = suzm.get_cities_for_zipcode(
zipcode, city_type=suzm.PRIMARY_CITY_TYPE)
acceptable_cities = suzm.get_cities_for_zipcode(
zipcode, city_type=suzm.ACCEPTABLE_CITY_TYPE)
not_acceptable_cities = suzm.get_cities_for_zipcode(
zipcode, city_type=suzm.NOT_ACCEPTABLE_CITY_TYPE)
zipcode_county = suzm.get_county_for_zipcode(zipcode)
zipcode_state = suzm.get_state_for_zipcode(zipcode)
zipcode_latitude = suzm.get_latitude_for_zipcode(zipcode)
zipcode_longitude = suzm.get_longitude_for_zipcode(zipcode)
zipcode_areacodes = suzm.get_areacodes_for_zipcode(zipcode)
zipcode_active_status = suzm.is_zipcode_active(zipcode)
zipcode_type = suzm.get_type_for_zipcode(zipcode)
"""
501 603 604 8720 8732 8753 9001 9203 9204 9213 11708
"""
if (DEBUG_PROC_ZC_ATTRS):
print("primary_cities", primary_cities)
if (not (acceptable_cities is None)):
print("acceptable_cities", len(acceptable_cities),
acceptable_cities)
else:
print("acceptable_cities", acceptable_cities)
if (not (not_acceptable_cities is None)):
print("not_acceptable_cities", len(not_acceptable_cities),
not_acceptable_cities)
else:
print("not_acceptable_cities", not_acceptable_cities)
print("zipcode_county", zipcode_county)
print("zipcode_state", zipcode_state)
print("zipcode_latitude", type(zipcode_latitude), zipcode_latitude)
print("zipcode_longitude", type(zipcode_longitude), zipcode_longitude)
print("zipcode_areacodes", type(zipcode_areacodes), zipcode_areacodes)
print("zipcode_active_status", zipcode_active_status)
print("zipcode_type", zipcode_type)
if ((not (zipcode_active_status)) and (primary_cities is None)):
zipattrsRows.append(["Current Status", "Zipcode is Invalid"])
else:
if (zipcode_active_status):
zipattrsRows.append(["Current Status", "Active"])
else:
zipattrsRows.append(["Current Status", "Decommissioined"])
if (primary_cities is None):
zipattrsRows.append(["Primary City", "None"])
else:
if ((primary_cities == suzm.APO_ZIPCODE_TYPE)
or (primary_cities == suzm.FPO_ZIPCODE_TYPE)
or (primary_cities == suzm.DPO_ZIPCODE_TYPE)):
zipattrsRows.append(["Primary City", "Washington DC"])
else:
zipattrsRows.append(["Primary City", str(primary_cities)])
if (zipcode_county is None):
zipattrsRows.append(["County", "None"])
else:
zipattrsRows.append(["County", str(zipcode_county)])
if (zipcode_state is None):
zipattrsRows.append(["State", "None"])
else:
zipattrsRows.append(["State", str(zipcode_state)])
if ((zipcode_latitude is None) or (zipcode_longitude is None)
or (numpy.isnan(zipcode_latitude))
or (numpy.isnan(zipcode_longitude))):
zipattrsRows.append(["[Latitude,Longitude]", "Unknown"])
else:
zipattrsRows.append([
"[Latitude,Longitude]", "[" + str(round(zipcode_latitude, 7)) +
" , " + str(round(zipcode_longitude, 7)) + "]"
])
if (zipcode_type is None):
zipattrsRows.append(["Zipcode Type", "None"])
else:
if (zipcode_type == suzm.UNIQUE_ZIPCODE_TYPE):
zipattrsRows.append(["Zipcode Type", str(suzm.UNIQUE_text)])
elif (zipcode_type == suzm.STANDARD_ZIPCODE_TYPE):
zipattrsRows.append(["Zipcode Type", str(suzm.STANDARD_text)])
elif (zipcode_type == suzm.PO_BOX_ZIPCODE_TYPE):
zipattrsRows.append(["Zipcode Type", str(suzm.PO_BOX_text)])
elif (zipcode_type == suzm.APO_ZIPCODE_TYPE):
zipattrsRows.append(["Zipcode Type", str(suzm.APO_text)])
elif (zipcode_type == suzm.FPO_ZIPCODE_TYPE):
zipattrsRows.append(["Zipcode Type", str(suzm.FPO_text)])
elif (zipcode_type == suzm.DPO_ZIPCODE_TYPE):
zipattrsRows.append(["Zipcode Type", str(suzm.DPO_text)])
else:
zipattrsRows.append(["Zipcode Type", "Unknown"])
if (not (zipcode_areacodes is None)):
if (type(zipcode_areacodes) == list):
for i in range(len(zipcode_areacodes)):
zipcode_areacodes[i] = zipcode_areacodes[i].replace(
"'", "")
zipattrsRows.append(["Area Codes", str(zipcode_areacodes)])
if (not (acceptable_cities is None)):
if (len(acceptable_cities) > 0):
zipattrsRows.append(
["Acceptable Cities",
str(acceptable_cities)])
if (not (not_acceptable_cities is None)):
if (len(not_acceptable_cities) > 0):
zipattrsRows.append(
["Not Acceptable Cities",
str(not_acceptable_cities)])
zipattrs_table = None
from dfcleanser.common.table_widgets import dcTable, get_row_major_table, ROW_MAJOR, SCROLL_DOWN
zipattrs_table = dcTable("Zipcode " + str(zipcode) + " Properties",
'zipcodeattrsid', cfg.SWZipcodeUtility_ID,
zipattrsHeader, zipattrsRows, zipattrsWidths,
zipattrsAligns)
zipattrs_table.set_small(True)
zipattrs_table.set_checkLength(False)
zipattrs_table.set_border(True)
zipattrs_table.set_tabletype(ROW_MAJOR)
zipattrs_table.set_rowspertable(50)
zipattrsHtml = get_row_major_table(zipattrs_table, SCROLL_DOWN, False)
gridclasses = ["dfc-top"]
gridhtmls = [zipattrsHtml]
display_generic_grid("display-geocode-coords-wrapper", gridclasses,
gridhtmls)
def display_dict_maint(keyValue=None, loadfile=None):
"""
* ------------------------------------------------------------------------
* function : display the user dicts maintenance form
*
* parms :
* keyValue - dict name
*
* -------------------------------------------------------------------------
"""
#print("display_dict_maint",keyValue,loadfile)
opstat = opStatus()
if (loadfile is None):
dict_maint_input_form = InputForm(
maint_dict_utility_input_id, maint_dict_utility_input_idList,
maint_dict_utility_input_labelList,
maint_dict_utility_input_typeList,
maint_dict_utility_input_placeholderList,
maint_dict_utility_input_jsList, maint_dict_utility_input_reqList)
else:
dict_maint_input_form = InputForm(
maint_dict_file_utility_input_id,
maint_dict_file_utility_input_idList,
maint_dict_file_utility_input_labelList,
maint_dict_file_utility_input_typeList,
maint_dict_file_utility_input_placeholderList,
maint_dict_file_utility_input_jsList,
maint_dict_file_utility_input_reqList)
selectDicts = []
from dfcleanser.sw_utilities.sw_utility_model import get_dicts_names, USER_CREATED
dict_names = get_dicts_names(USER_CREATED)
#print("dict_names",dict_names)
if (not (dict_names is None)):
if (keyValue is None):
def_dict = dict_names[0]
else:
def_dict = keyValue
seldict = swum.get_Dict(def_dict, USER_CREATED)
keys = list(seldict.keys())
if ((def_dict == "Country_Codes") or (def_dict == "Language_Codes")):
keys.sort()
seldict = swum.get_pretty_dict(seldict, keys)
else:
dict_names = ["No User dicts defined"]
def_dict = "No User dicts defined"
seldict = "User defined dict"
dictssel = {
"default": def_dict,
"list": dict_names,
"callback": "select_dict"
}
selectDicts.append(dictssel)
from dfcleanser.common.common_utils import get_select_defaults
get_select_defaults(dict_maint_input_form, maint_dict_utility_input_id,
maint_dict_utility_input_idList,
maint_dict_utility_input_typeList, selectDicts)
dict_maint_input_form.set_gridwidth(700)
#dict_maint_input_form.set_custombwidth(110)
if (loadfile is None):
dict_maint_input_form.set_buttonstyle({
"font-size": 13,
"height": 75,
"width": 90,
"left-margin": 20
})
else:
dict_maint_input_form.set_buttonstyle({
"font-size": 13,
"height": 75,
"width": 90,
"left-margin": 205
})
dict_maint_input_form.set_fullparms(True)
cfg.drop_config_value(maint_dict_utility_input_id + "Parms")
cfg.drop_config_value(maint_dict_utility_input_id + "ParmsProtect")
if (not (loadfile is None)):
import json
#from dfcleanser.common.common_utils import does_file_exist
#print("does_file_exist",does_file_exist(loadfile))
try:
with open(loadfile, 'r') as ds_file:
ds = json.load(ds_file)
ds_file.close()
keys = list(ds.keys())
seldict = swum.get_pretty_dict(ds, keys)
#print(seldict)
except Exception as e:
opstat.set_errorMsg("invalid user file to load " + loadfile)
opstat.set_exception(e)
if (opstat.get_status()):
if (loadfile is None):
cfg.set_config_value(maint_dict_utility_input_id + "Parms",
[def_dict, "", seldict, ""])
else:
cfg.set_config_value(maint_dict_utility_input_id + "Parms",
[def_dict, "", seldict, loadfile])
cfg.set_config_value(maint_dict_utility_input_id + "ParmsProtect",
[True, False, True, True])
help_note = "To add a user dict enter parms and values above and click on 'Add User Dict'.</br>To update the current dict change values and click on 'Update User Dict'"
from dfcleanser.common.common_utils import get_help_note_html
dict_maint_notes_html = get_help_note_html(help_note, 80, 75, None)
dict_maint_html = "Fill in new user dict parms or update currently displayed user dict."
dict_maint_html = dict_maint_input_form.get_html()
dict_maint_title_html = "<div>User Dicts</div><br></br>"
gridclasses = [
"dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"
]
gridhtmls = [
dict_maint_title_html, dict_maint_html, dict_maint_notes_html
]
#print(dict_maint_html)
#print(dict_maint_notes_html)
print("\n")
display_generic_grid("sw-utils-listdict-wrapper", gridclasses,
gridhtmls)
else:
display_exception(opstat)
add_error_to_log("[Get User Dict from File] " + loadfile +
str(sys.exc_info()[0].__name__))
def display_dc_data_scripting(optionId, parms=None):
from IPython.display import clear_output
clear_output()
from dfcleanser.common.cfg import check_if_dc_init
if (not check_if_dc_init()):
from dfcleanser.common.display_utils import display_dfcleanser_taskbar
display_dfcleanser_taskbar(
ButtonGroupForm(dc_script_tb_id, dc_script_tb_keyTitleList,
dc_script_tb_jsList, dc_script_tb_centered), False)
from dfcleanser.scripting.data_scripting_control import clear_data_scripting_data
clear_data_scripting_data()
return
from dfcleanser.common.display_utils import display_dfcleanser_taskbar
display_dfcleanser_taskbar(
ButtonGroupForm(dc_script_tb_id, dc_script_tb_keyTitleList,
dc_script_tb_jsList, dc_script_tb_centered), False)
if (parms == None):
from dfcleanser.scripting.data_scripting_control import clear_data_scripting_data
clear_data_scripting_data()
else:
funcid = int(parms[0])
if (funcid == dsm.TURN_ON_SCRIPTING):
from dfcleanser.scripting.data_scripting_control import set_scripting_status
set_scripting_status(True)
from dfcleanser.common.common_utils import display_status_note
display_status_note("Scripting is turned on")
elif (funcid == dsm.TURN_OFF_SCRIPTING):
from dfcleanser.scripting.data_scripting_control import set_scripting_status
set_scripting_status(False)
from dfcleanser.common.common_utils import display_status_note
display_status_note("Scripting is turned off")
elif (funcid == dsm.SHOW_CURRENT_SCRIPT):
display_scripting_forms()
elif (funcid == dsm.ADD_TO_CURRENT_SCRIPT):
script_form = InputForm(
dc_add_code_input_id, dc_add_code_input_idList,
dc_add_code_input_labelList, dc_add_code_input_typeList,
dc_add_code_input_placeholderList, dc_add_code_input_jsList,
dc_add_code_input_reqList)
script_form.set_shortForm(True)
script_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 140,
"left-margin": 70
})
script_form.set_gridwidth(880)
script_form.set_fullparms(True)
script_form_html = script_form.get_html()
script_title_html = "<div>Scripting</div><br>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"]
gridhtmls = [script_title_html, script_form_html]
from dfcleanser.common.common_utils import display_generic_grid
display_generic_grid("data-scripting-wrapper", gridclasses,
gridhtmls)
elif (funcid == dsm.ADD_CODE_SCRIPT):
from dfcleanser.scripting.data_scripting_control import add_code_to_script
add_code_to_script(parms)
display_status("Code added to Current Script succesfully ")
elif (funcid == dsm.DELETE_CURRENT_SCRIPT):
from dfcleanser.scripting.data_scripting_control import drop_current_script
drop_current_script()
display_status("Current Script Cleared succesfully ")
elif (funcid == dsm.LOAD_BACKUP_SCRIPT):
from dfcleanser.scripting.data_scripting_control import load_backup_scriptlog_to_current
load_backup_scriptlog_to_current()
display_scripting_forms()
display_status("Current Script Loaded from Backup ")
elif (funcid == dsm.SAVE_BACKUP_SCRIPT):
codeparms = get_parms_for_input(parms[1], dc_script_input_idList)
save_code = get_code_from_form(codeparms)
from dfcleanser.scripting.data_scripting_control import (
set_current_scriptlog, save_current_scriptlog_to_backup)
set_current_scriptlog(save_code)
save_current_scriptlog_to_backup(save_code)
display_scripting_forms()
display_status("Current Script Backed up successfully ")
elif (funcid == dsm.RUN_CURRENT_SCRIPT):
opstat = opStatus()
from dfcleanser.scripting.data_scripting_control import run_scriptlog
run_scriptlog(parms, opstat)
if (opstat.get_status()):
display_status("Current Script Run successfully ")
else:
display_exception(opstat)
def display_inspect_cols(parms):
"""
* --------------------------------------------------------------------------
* function : display the inspect cols option
*
* parms :
*
* returns :
* N/A
* --------------------------------------------------------
"""
opstat = opStatus()
clock = RunningClock()
clock.start()
try:
df = cfg.get_current_chapter_df(cfg.DataInspection_ID)
colnames = df.columns.tolist()
if (not (parms is None)):
colname = parms
else:
colname = colnames[0]
cnames = {
'default': colname,
'list': colnames,
"callback": "change_inspect_cols_col",
"size": 10
}
if (is_numeric_col(df, colname)):
coldetails_form = InputForm(diw.inspect_col_input_id,
diw.inspect_col_input_idList,
diw.inspect_col_input_labelList,
diw.inspect_col_input_typeList,
diw.inspect_col_input_placeholderList,
diw.inspect_col_input_jsList,
diw.inspect_col_input_reqList)
else:
coldetails_form = InputForm(
diw.inspect_nn_col_input_id, diw.inspect_nn_col_input_idList,
diw.inspect_nn_col_input_labelList,
diw.inspect_nn_col_input_typeList,
diw.inspect_nn_col_input_placeholderList,
diw.inspect_nn_col_input_jsList,
diw.inspect_nn_col_input_reqList)
selectDicts = []
selectDicts.append(cnames)
get_select_defaults(coldetails_form, diw.inspect_col_input_id,
diw.inspect_col_input_idList,
diw.inspect_col_input_typeList, selectDicts)
coldetails_form.set_shortForm(True)
coldetails_form.set_fullparms(True)
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
coldetails_form.set_gridwidth(360)
if (is_numeric_col(df, colname)):
coldetails_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 85,
"left-margin": 2
})
else:
coldetails_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 85,
"left-margin": 75
})
else:
coldetails_form.set_gridwidth(480)
if (is_numeric_col(df, colname)):
coldetails_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 110,
"left-margin": 2
})
else:
coldetails_form.set_buttonstyle({
"font-size": 12,
"height": 75,
"width": 110,
"left-margin": 110
})
coldetails_html = coldetails_form.get_html()
from dfcleanser.data_cleansing.data_cleansing_widgets import display_col_stats
col_stats_html = display_col_stats(df, colname, False, True)
gridclasses = ["dfc-left", "dfc-right"]
gridhtmls = [col_stats_html, coldetails_html]
if (cfg.get_dfc_mode() == cfg.INLINE_MODE):
display_generic_grid("df-inspection-column-data-wrapper",
gridclasses, gridhtmls)
else:
display_generic_grid("df-inspection-pop-up-column-data-wrapper",
gridclasses, gridhtmls)
except Exception as e:
opstat.store_exception("Error displaying column data\n ", e)
clock.stop()
if (not (opstat.get_status())):
display_exception(opstat)
def display_process_subset():
"""
* --------------------------------------------------------------------------
* function : display current df subset form
*
* parms :
* df - dataframe to subset from
*
* returns : N/A
* --------------------------------------------------------
"""
current_step = swsm.get_current_subset_step()
dftitle = current_step.get_output_subset_df_title()
subset_df = swsm.get_current_subset_df()
df_stats_table = get_df_stats_table(dftitle, subset_df)
df_cols_table = get_column_stats_table(dftitle, subset_df)
from dfcleanser.common.html_widgets import InputForm
subset_input_form = InputForm(
get_subset_run_input_id, get_subset_run_input_idList,
get_subset_run_input_labelList, get_subset_run_input_typeList,
get_subset_run_input_placeholderList, get_subset_run_input_jsList,
get_subset_run_input_reqList)
selectDicts = []
colnames = subset_df.columns.tolist()
cols_name_list = [" "]
for i in range(len(colnames)):
cols_name_list.append(colnames[i])
cnames = {
"default": cols_name_list[0],
"list": cols_name_list,
"callback": "change_run_subset_cols"
}
selectDicts.append(cnames)
subssel = {"default": "Keep", "list": ["Keep", "Drop"]}
selectDicts.append(subssel)
savesel = {"default": "True", "list": ["True", "False"]}
selectDicts.append(savesel)
get_select_defaults(subset_input_form, get_subset_run_input_form[0],
get_subset_run_input_form[1],
get_subset_run_input_form[3], selectDicts)
subset_input_form.set_shortForm(False)
subset_input_form.set_gridwidth(680)
subset_input_form.set_custombwidth(90)
subset_input_form.set_fullparms(True)
cfg.set_config_value(get_subset_run_input_id + "Parms",
[dftitle, "", "", ""])
cfg.set_config_value(get_subset_run_input_id + "ParmsProtect",
[True, False, False, False])
get_subset_input_html = subset_input_form.get_html()
get_subset_heading_html = "<div>Process Dataframe Subset</div><br>"
gridclasses = [
"dfc-top", "dfc-main", "dfcleanser-common-grid-header", "dfc-bottom"
]
gridhtmls = [
df_stats_table, df_cols_table, get_subset_heading_html,
get_subset_input_html
]
print("\n")
display_generic_grid("sw-utils-subset-run-wrapper", gridclasses, gridhtmls)
def process_zipcode_cities(parms):
opstat = opStatus()
fparms = get_parms_for_input(parms, suzw.zipcode_cities_input_idList)
city = fparms[0]
state = fparms[1][:2]
cfg.set_config_value(suzw.zipcode_cities_input_id + "Parms", fparms)
suzw.display_get_zips_for_city(parms)
print("\n")
cityzipsHeader = [""]
cityzipsRows = []
cityzipsWidths = [30, 70]
cityzipsAligns = ["left", "left"]
cityzips = suzm.get_zipcodes_for_city(
state,
city,
zipcode_type=suzm.STANDARD_ZIPCODE_TYPE,
active_status=suzm.ACTIVE_STATUS_TYPE)
citypobzips = suzm.get_zipcodes_for_city(
state,
city,
zipcode_type=suzm.PO_BOX_ZIPCODE_TYPE,
active_status=suzm.ACTIVE_STATUS_TYPE)
cityuniquezips = suzm.get_zipcodes_for_city(
state,
city,
zipcode_type=suzm.UNIQUE_ZIPCODE_TYPE,
active_status=suzm.ACTIVE_STATUS_TYPE)
citydecomzips = suzm.get_zipcodes_for_city(
state,
city,
zipcode_type=suzm.ANY_ZIPCODE_TYPE,
active_status=suzm.DECOMMISIONED_STATUS_TYPE)
if (not (cityzips is None)):
cityzipsRows.append([suzm.STANDARD_text + " Zipcodes", str(cityzips)])
if (not (citypobzips is None)):
cityzipsRows.append([suzm.PO_BOX_text + " Zipcodes", str(citypobzips)])
if (not (cityuniquezips is None)):
cityzipsRows.append(
[suzm.UNIQUE_text + " Zipcodes",
str(cityuniquezips)])
if (not (citydecomzips is None)):
cityzipsRows.append(
["Decommissioned" + " Zipcodes",
str(citydecomzips)])
cityzips_table = None
from dfcleanser.common.table_widgets import dcTable, get_row_major_table, ROW_MAJOR, SCROLL_DOWN
cityzips_table = dcTable(
"Zipcodes For " + str(city.upper()) + ", " + str(state),
'cityzipcodesid', cfg.SWZipcodeUtility_ID, cityzipsHeader,
cityzipsRows, cityzipsWidths, cityzipsAligns)
cityzips_table.set_small(True)
cityzips_table.set_checkLength(False)
cityzips_table.set_border(True)
cityzips_table.set_tabletype(ROW_MAJOR)
cityzips_table.set_rowspertable(50)
cityzipsHtml = get_row_major_table(cityzips_table, SCROLL_DOWN, False)
gridclasses = ["dfc-top"]
gridhtmls = [cityzipsHtml]
display_generic_grid("display-geocode-coords-wrapper", gridclasses,
gridhtmls)
def display_funcs():
"""
* ------------------------------------------------------------------------
* function : display the dfc funcs start form
*
* parms :
*
* -------------------------------------------------------------------------
"""
list_funcs_input_form = InputForm(
build_funcs_utility_input_id, build_funcs_utility_input_idList,
build_funcs_utility_input_labelList,
build_funcs_utility_input_typeList,
build_funcs_utility_input_placeholderList,
build_funcs_utility_input_jsList, build_funcs_utility_input_reqList)
selectDicts = []
from dfcleanser.sw_utilities.sw_utility_genfunc_model import get_genfunc_list
gen_funcs = get_genfunc_list()
default_func = gen_funcs[0]
funcssel = {
"default": default_func,
"list": gen_funcs,
"callback": "select_gen_func"
}
selectDicts.append(funcssel)
from dfcleanser.common.common_utils import get_select_defaults
get_select_defaults(list_funcs_input_form, build_funcs_utility_input_id,
build_funcs_utility_input_idList,
build_funcs_utility_input_typeList, selectDicts)
list_funcs_input_form.set_gridwidth(700)
list_funcs_input_form.set_custombwidth(120)
list_funcs_input_form.set_fullparms(True)
from dfcleanser.sw_utilities.sw_utility_genfunc_model import reservedfunctionsmodule
from dfcleanser.sw_utilities.sw_utility_genfunc_functions import get_generic_function_code
func_code = get_generic_function_code(default_func)
cfg.set_config_value(build_funcs_utility_input_id + "Parms",
[default_func, reservedfunctionsmodule, func_code])
cfg.set_config_value(build_funcs_utility_input_id + "ParmsProtect",
[False, True, True])
help_note = "Generic functions help note'"
from dfcleanser.common.common_utils import get_help_note_html
list_funcs_notes_html = get_help_note_html(help_note, 100, None, None)
list_funcs_html = ""
list_funcs_html = list_funcs_input_form.get_html()
list_funcs_title_html = "<div>Generic dfc Functions</div><br></br>"
gridclasses = ["dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"]
gridhtmls = [list_funcs_title_html, list_funcs_html, list_funcs_notes_html]
print("\n")
display_generic_grid("sw-utils-listdict-wrapper", gridclasses, gridhtmls)
def display_manual_df_subset_setup(sequence_title, input_df_title,
colnames_list, cols_action, stepid):
"""
* --------------------------------------------------------------------------
* function : display current manual df subset form
*
* parms :
* df - dataframe to subset from
* filters - filters form
* colname - filters column name
*
* returns : N/A
* --------------------------------------------------------
"""
df = swsm.get_current_subset_df()
col_stats_table = get_column_stats_table(input_df_title, df)
from dfcleanser.common.html_widgets import InputForm
subset_input_form = InputForm(get_manual_input_id, get_manual_input_idList,
get_manual_input_labelList,
get_manual_input_typeList,
get_manual_input_placeholderList,
get_manual_input_jsList,
get_manual_input_reqList)
selectDicts = []
dataframes = {
"default": input_df_title,
"list": [input_df_title]
} #cfg.get_dfc_dataframes_select_list(cfg.SWDFSubsetUtility_ID)
selectDicts.append(dataframes)
current_df = df #cfg.get_dfc_dataframe_df(input_df_title)
colnames = current_df.columns.tolist()
cols_name_list = [" "]
for i in range(len(colnames)):
cols_name_list.append(colnames[i])
cnames = {
"default": cols_name_list[0],
"list": cols_name_list,
"callback": "change_subset_cols"
}
selectDicts.append(cnames)
if (cols_action == "Keep"):
subssel = {"default": "Keep", "list": ["Keep", "Drop"]}
else:
subssel = {"default": "Drop", "list": ["Keep", "Drop"]}
selectDicts.append(subssel)
get_select_defaults(subset_input_form, get_manual_input_form[0],
get_manual_input_form[1], get_manual_input_form[3],
selectDicts)
if (len(colnames_list) > 0):
cnames = str(colnames_list)
else:
cnames = ""
cfg.set_config_value(get_manual_input_id + "Parms",
["", cnames, "", cols_action])
cfg.set_config_value(get_manual_input_id + "ParmsProtect",
[True, False, False, False])
subset_input_form.set_shortForm(False)
subset_input_form.set_gridwidth(680)
subset_input_form.set_custombwidth(140)
subset_input_form.set_fullparms(True)
get_subset_input_html = subset_input_form.get_html()
get_subset_heading_html = "<div>Run Subset Sequence '" + str(
sequence_title) + "' Step " + str(stepid) + "</div><br></br>"
gridclasses = ["dfc-top", "dfcleanser-common-grid-header", "dfc-bottom"]
gridhtmls = [
col_stats_table, get_subset_heading_html, get_subset_input_html
]
print("\n")
display_generic_grid("sw-utils-subset-wrapper", gridclasses, gridhtmls)
def display_list_maint(keyValue=None, loadfile=None):
"""
* ------------------------------------------------------------------------
* function : display the user lists maintenance form
*
* parms :
* keyValue - list name
*
* -------------------------------------------------------------------------
"""
opstat = opStatus()
if (loadfile is None):
list_maint_input_form = InputForm(
maint_list_utility_input_id, maint_list_utility_input_idList,
maint_list_utility_input_labelList,
maint_list_utility_input_typeList,
maint_list_utility_input_placeholderList,
maint_list_utility_input_jsList, maint_list_utility_input_reqList)
else:
list_maint_input_form = InputForm(
maint_list_file_utility_input_id,
maint_list_file_utility_input_idList,
maint_list_file_utility_input_labelList,
maint_list_file_utility_input_typeList,
maint_list_file_utility_input_placeholderList,
maint_list_file_utility_input_jsList,
maint_list_file_utility_input_reqList)
selectDicts = []
from dfcleanser.sw_utilities.sw_utility_model import get_lists_names, USER_CREATED
list_names = get_lists_names(USER_CREATED)
#print("list_names",list_names)
if (not (list_names is None)):
if (keyValue is None):
def_list = list_names[0]
else:
def_list = keyValue
sellist = swum.get_List(def_list, USER_CREATED)
dsstr = "["
for i in range(len(sellist)):
dsstr = dsstr + str(sellist[i])
if (i == (len(sellist) - 1)):
dsstr = dsstr + "]"
else:
dsstr = dsstr + ","
else:
list_names = ["No User lists defined"]
def_list = "No User lists defined"
sellist = "User defined list"
listssel = {
"default": def_list,
"list": list_names,
"callback": "select_list"
}
selectDicts.append(listssel)
from dfcleanser.common.common_utils import get_select_defaults
get_select_defaults(list_maint_input_form, maint_list_utility_input_id,
maint_list_utility_input_idList,
maint_list_utility_input_typeList, selectDicts)
list_maint_input_form.set_gridwidth(700)
if (loadfile is None):
list_maint_input_form.set_buttonstyle({
"font-size": 13,
"height": 75,
"width": 90,
"left-margin": 20
})
else:
list_maint_input_form.set_buttonstyle({
"font-size": 13,
"height": 75,
"width": 90,
"left-margin": 205
})
list_maint_input_form.set_fullparms(True)
cfg.drop_config_value(maint_list_utility_input_id + "Parms")
cfg.drop_config_value(maint_list_utility_input_id + "ParmsProtect")
if (not (loadfile is None)):
import json
from dfcleanser.common.common_utils import does_file_exist
if (does_file_exist(loadfile)):
try:
with open(loadfile, 'r') as ds_file:
ds = json.load(ds_file)
ds_file.close()
dsstr = "["
for i in range(len(ds)):
dsstr = dsstr + str(ds[i])
if (i == (len(ds) - 1)):
dsstr = dsstr + "]"
else:
dsstr = dsstr + ","
except Exception as e:
opstat.set_status(False)
opstat.set_errorMsg("Error processing user file to load" +
loadfile)
opstat.set_exception(e)
else:
opstat.set_status(False)
opstat.set_errorMsg("invalid user file to load" + loadfile)
if (opstat.get_status()):
if (loadfile is None):
cfg.set_config_value(maint_list_utility_input_id + "Parms",
[def_list, "", dsstr, ""])
else:
cfg.set_config_value(maint_list_utility_input_id + "Parms",
[def_list, "", dsstr, loadfile])
cfg.set_config_value(maint_list_utility_input_id + "ParmsProtect",
[True, False, True, True])
help_note = "To add a user list enter parms and values above and click on 'Add User List'.</br>To update the current list change values and click on 'Update User List'"
from dfcleanser.common.common_utils import get_help_note_html
list_maint_notes_html = get_help_note_html(help_note, 80, 75, None)
list_maint_html = ""
list_maint_html = list_maint_input_form.get_html()
list_maint_title_html = "<div>User Lists</div><br></br>"
gridclasses = [
"dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"
]
gridhtmls = [
list_maint_title_html, list_maint_html, list_maint_notes_html
]
#print(list_maint_html)
#print(list_maint_notes_html)
print("\n")
display_generic_grid("sw-utils-listdict-wrapper", gridclasses,
gridhtmls)
else:
display_exception(opstat)
add_error_to_log("[Get User Dict from File] " + loadfile +
str(sys.exc_info()[0].__name__))
def display_dc_pandas_export_sql_inputs(fId,
dbId,
dbconparms,
exportparms=None):
"""
* --------------------------------------------------------------------------
* function : display pandas sql export form
*
* parms :
* fid - export type
* dbid - database id
* dbconparms - db connector parms
* exportparms - export parms
*
* returns : N/A
* --------------------------------------------------------
"""
opstat = opStatus()
opstatStatus = True
listHtml = ""
dbid = int(dbId)
fid = int(fId)
fparms = None
if (fid == 0):
dbid = cfg.get_config_value(cfg.CURRENT_DB_ID_KEY)
from dfcleanser.data_import.data_import_widgets import (
get_table_names, TABLE_NAMES, get_rows_html)
tablelist = get_table_names(dbid, opstat)
listHtml = get_rows_html(tablelist, TABLE_NAMES, True)
elif (fid == 1):
fparms = get_parms_for_input(exportparms,
pandas_export_sqltable_idList)
dbid = cfg.get_config_value(cfg.CURRENT_DB_ID_KEY)
cfg.set_config_value(pandas_export_sqltable_id + "Parms", fparms)
if (len(fparms[0]) > 0):
from dfcleanser.data_import.data_import_widgets import (
get_column_names, get_rows_html)
from dfcleanser.data_import.data_import_model import COLUMN_NAMES
columnlist = get_column_names(dbid, fparms[1], opstat)
listHtml = get_rows_html(columnlist, COLUMN_NAMES, True)
else:
opstat.set_status(False)
opstat.set_errorMsg("No Table Selected")
elif (fid == 2):
cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbid)
dbcondict = {}
if (not (dbconparms == None)):
parse_connector_parms(dbconparms, dbid, dbcondict)
else:
conparms = get_stored_con_Parms(dbid)
parse_connector_parms(conparms, dbid, dbcondict)
listHtml = get_db_connector_list(dbid, dbcondict)
if (not (opstat.get_status())):
dbcondict = {}
conparms = get_stored_con_Parms(dbid)
parse_connector_parms(conparms, dbid, dbcondict)
listHtml = get_db_connector_list(dbid, dbcondict)
opstatStatus = opstat.get_status()
opstatErrormsg = opstat.get_errorMsg()
opstat.set_status(True)
if (opstat.get_status()):
export_sql_input_form = InputForm(
pandas_export_sqltable_id, pandas_export_sqltable_idList,
pandas_export_sqltable_labelList, pandas_export_sqltable_typeList,
pandas_export_sqltable_placeholderList,
pandas_export_sqltable_jsList, pandas_export_sqltable_reqList)
selectDicts = []
df_list = cfg.get_dfc_dataframes_select_list(cfg.DataExport_ID)
selectDicts.append(df_list)
exists = {"default": "fail", "list": ["fail", "replace", "append"]}
selectDicts.append(exists)
index = {"default": "True", "list": ["True", "False"]}
selectDicts.append(index)
get_select_defaults(export_sql_input_form, pandas_export_sqltable_id,
pandas_export_sqltable_idList,
pandas_export_sqltable_typeList, selectDicts)
export_sql_input_form.set_shortForm(False)
export_sql_input_form.set_gridwidth(680)
export_sql_input_form.set_custombwidth(125)
export_sql_input_form.set_fullparms(True)
export_sql_input_html = ""
export_sql_input_html = export_sql_input_form.get_html()
export_sql_heading_html = "<div>" + get_pandas_export_input_title(
dem.SQLTABLE_EXPORT, dbid) + "</div><br>"
if (not (exportparms == None)):
cfg.set_config_value(pandas_export_sqltable_id + "Parms", fparms)
gridclasses = [
"dfcleanser-common-grid-header", "dfc-left", "dfc-right"
]
gridhtmls = [export_sql_heading_html, listHtml, export_sql_input_html]
display_generic_grid("data-import-sql-table-wrapper", gridclasses,
gridhtmls)
if (not (opstatStatus)):
opstat.set_status(opstatStatus)
opstat.set_errorMsg(opstatErrormsg)
display_exception(opstat)
