def display_df_dataframes(title): """ * -------------------------------------------------------------------------- * function : display dataframe manager form * * parms : * N/A * * returns : N/A * -------------------------------------------------------- """ dfmgr_input_html = get_df_list_html(title) dfmgr_heading_html = "<div>Current dfCleanser dataframes </div>" gridclasses = ["dfcleanser-common-grid-header", "dfc-main"] gridhtmls = [dfmgr_heading_html, dfmgr_input_html] print("\n") if (cfg.get_dfc_mode() == cfg.INLINE_MODE): display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses, gridhtmls) else: display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses, gridhtmls, True) from dfcleanser.common.display_utils import display_pop_up_buffer display_pop_up_buffer()
def display_system_main_taskbar(): if (cfg.get_dfc_mode() == cfg.INLINE_MODE): from dfcleanser.common.display_utils import display_dfcleanser_taskbar display_dfcleanser_taskbar( ButtonGroupForm(system_environment_doc_id, system_environment_keyTitleList, system_environment_jsList, system_environment_centered)) else: system_tb_A = ButtonGroupForm(system_environmentA_doc_id, system_environmentA_keyTitleList, system_environmentA_jsList, system_environmentA_centered) system_tb_A.set_gridwidth(480) system_tb_A_html = system_tb_A.get_html() system_tb_B = ButtonGroupForm(system_environmentB_doc_id, system_environmentB_keyTitleList, system_environmentB_jsList, system_environmentB_centered) system_tb_B.set_gridwidth(480) system_tb_B_html = system_tb_B.get_html() gridclasses = ["dfc-top-", "dfc-footer"] gridhtmls = [system_tb_A_html, system_tb_B_html] display_generic_grid("dfcleanser-system-tb-pop-up-wrapper", gridclasses, gridhtmls)
def display_get_zipcode_attributes(parms): """ * --------------------------------------------------------- * function : display the calculate distance form * * parms : * pytype - address or coords * * returns : * N/A * -------------------------------------------------------- """ zip_attr_form = InputForm( zipcode_atributes_input_id, zipcode_atributes_input_idList, zipcode_atributes_input_labelList, zipcode_atributes_input_typeList, zipcode_atributes_input_placeholderList, zipcode_atributes_input_jsList, zipcode_atributes_input_reqList) zip_attr_form.set_gridwidth(720) zip_attr_form.set_custombwidth(100) zip_attr_form.set_fullparms(True) zip_attr_input_html = "" zip_attr_input_html = zip_attr_form.get_html() zip_attr_heading_html = "<div>Get Zipcode Attributes</div>" gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"] gridhtmls = [zip_attr_heading_html, zip_attr_input_html] print("\n") display_generic_grid("geocode-utility-wrapper", gridclasses, gridhtmls)
def display_inspect_outliers(colname): """ * -------------------------------------------------------------------------- * function : display the inspect outliers option * * parms : * * returns : * N/A * -------------------------------------------------------- """ opstat = opStatus() display_inspect_cols(colname) outliers_html = diw.get_simple_outliers(cfg.get_current_chapter_df( cfg.DataInspection_ID), colname, opstat, display=False) gridclasses = ["dfc-main"] gridhtmls = [outliers_html] print("\n") if (cfg.get_dfc_mode() == cfg.INLINE_MODE): display_generic_grid("df-inspection-outliers-data-wrapper", gridclasses, gridhtmls) else: display_generic_grid("df-inspection-outliers-pop-up-data-wrapper", gridclasses, gridhtmls)
def display_df_criteria(df_title, df): """ * -------------------------------------------------------------------------- * function : display current df subset form * * parms : * df - dataframe to subset from * filters - filters form * colname - filters column name * * returns : N/A * -------------------------------------------------------- """ col_stats_table = get_column_stats_table(df_title, df) from dfcleanser.common.html_widgets import InputForm subset_input_form = InputForm(get_subset_criteria_input_id, get_subset_criteria_input_idList, get_subset_criteria_input_labelList, get_subset_criteria_input_typeList, get_subset_criteria_input_placeholderList, get_subset_criteria_input_jsList, get_subset_criteria_input_reqList) subset_input_form.set_label_note({ 2: [ dfchelp.SUBSET_CRITERIA, "dataframe subset criteria", "subsetcriteriaimageid" ] }) subset_input_form.set_textarea_resize_flag(1, False) subset_input_form.set_textarea_resize_flag(3, False) subset_input_form.set_shortForm(False) subset_input_form.set_gridwidth(620) subset_input_form.set_custombwidth(110) subset_input_form.set_fullparms(True) subset_input_form.set_custom_font_size("gsselectstring", 14) cfg.set_config_value(get_subset_criteria_input_id + "Parms", [ "", swsm.starting_criteria_preamble, swsm.starting_criteria, swsm.starting_criteria_postamble ]) cfg.set_config_value(get_subset_criteria_input_id + "ParmsProtect", [False, True, False, True]) get_subset_input_html = subset_input_form.get_html() get_subset_heading_html = "<div>Get Dataframe Subset</div><br></br>" gridclasses = ["dfc-top", "dfcleanser-common-grid-header", "dfc-bottom"] gridhtmls = [ col_stats_table, get_subset_heading_html, get_subset_input_html ] print("\n") display_generic_grid("sw-utils-subset-wrapper", gridclasses, gridhtmls)
def display_data_export_parms(title, plist, fparms, exportID, fname, custom=False, dbnote=False): """ * -------------------------------------------------------------------------- * function : display export parms * * parms : * title - poarms title * plist - label list * fparms - values list * exportID - export table id * fname - file name * dbnote - dbnote flag * * returns : N/A * -------------------------------------------------------- """ #print("display_data_export_parms",title,plist,fparms) parms_html = displayParms(title, plist, fparms, exportID, 100, 2, False) if (custom): status_html = display_status( "Custom export code Exported successfully", False, False) exportnotes = [ "[Total Export Time] : " + str(get_formatted_time(time.time() - get_data_export_start())) + " seconds", "( check if df exists via dfcleanser.common.cfg.is_a_dfc_dataframe_loaded() )" ] else: if (dbnote): status_html = display_status( " Dataframe Exported successfully to table " + fname, False, False) else: status_html = display_status( " Dataframe Exported successfully to File " + fname, False, False) exportnotes = [ "[Total Export Time] : " + str(get_formatted_time(time.time() - get_data_export_start())) + " seconds" ] notes_html = display_notes(exportnotes, False) gridclasses = ["dfc-top", "dfc-left", "dfc-right"] gridhtmls = [parms_html, status_html, notes_html] display_generic_grid("data-import-stats-wrapper", gridclasses, gridhtmls)
def display_saved_subset_sequences(): """ * -------------------------------------------------------------------------- * function : display current saved sequences * * parms : * df - dataframe to subset from * * returns : N/A * -------------------------------------------------------- """ from dfcleanser.common.html_widgets import InputForm subset_input_form = InputForm(get_subset_sequences_input_id, get_subset_sequences_input_idList, get_subset_sequences_input_labelList, get_subset_sequences_input_typeList, get_subset_sequences_input_placeholderList, get_subset_sequences_input_jsList, get_subset_sequences_input_reqList) selectDicts = [] sequences = swsm.get_sequences_list() if (len(sequences) > 0): seqdict = {"default": sequences[0], "list": sequences} else: seqdict = { "default": "no saved sequences", "list": ["no saved sequences"] } selectDicts.append(seqdict) subsetact = {"default": "Auto Run", "list": ["Auto Run", "Manual Run"]} selectDicts.append(subsetact) get_select_defaults(subset_input_form, get_subset_sequences_input_form[0], get_subset_sequences_input_form[1], get_subset_sequences_input_form[3], selectDicts) subset_input_form.set_shortForm(False) subset_input_form.set_gridwidth(680) subset_input_form.set_custombwidth(140) subset_input_form.set_fullparms(True) get_subset_input_html = subset_input_form.get_html() get_subset_heading_html = "<div>Run Saved Subset Sequence</div><br>" gridclasses = ["dfcleanser-common-grid-header", "dfc-main"] gridhtmls = [get_subset_heading_html, get_subset_input_html] print("\n") display_generic_grid("sw-utils-subset-main-wrapper", gridclasses, gridhtmls)
def display_sequence_save_subset(df_title, df, auto=False): """ * -------------------------------------------------------------------------- * function : display current df subset run * * parms : * df - dataframe to subset from * * returns : N/A * -------------------------------------------------------- """ df_stats_table = get_df_stats_table(df_title, df) df_cols_table = get_column_stats_table(df_title, df) from dfcleanser.common.html_widgets import InputForm subset_input_form = InputForm( get_saved_save_input_id, get_saved_save_input_idList, get_saved_save_input_labelList, get_saved_save_input_typeList, get_saved_save_input_placeholderList, get_saved_save_input_jsList, get_saved_save_input_reqList) selectDicts = [] dropsel = {"default": "False", "list": ["True", "False"]} selectDicts.append(dropsel) get_select_defaults(subset_input_form, get_saved_save_input_id, get_saved_save_input_idList, get_saved_save_input_typeList, selectDicts) subset_input_form.set_shortForm(False) subset_input_form.set_gridwidth(680) subset_input_form.set_custombwidth(140) subset_input_form.set_fullparms(True) cfg.set_config_value( get_saved_save_input_id + "Parms", [df_title, "./" + df_title + ".csv", df_title, "False"]) cfg.drop_config_value(get_saved_save_input_id + "ParmsProtect") get_subset_input_html = subset_input_form.get_html() get_subset_heading_html = "<div>Save Dataframe Subset</div><br>" gridclasses = [ "dfc-top", "dfc-main", "dfcleanser-common-grid-header", "dfc-bottom" ] gridhtmls = [ df_stats_table, df_cols_table, get_subset_heading_html, get_subset_input_html ] print("\n") display_generic_grid("sw-utils-subset-run-wrapper", gridclasses, gridhtmls)
def display_offline(): """ * -------------------------------------------------------------------------- * function : display offline toggle mode * * parms : * N/A * * returns : N/A * -------------------------------------------------------- """ from dfcleanser.common.html_widgets import InputForm dfmanager_input_form = InputForm( dfmgr_offline_input_id, dfmgr_offline_input_idList, dfmgr_offline_input_labelList, dfmgr_offline_input_typeList, dfmgr_offline_input_placeholderList, dfmgr_offline_input_jsList, dfmgr_offline_input_reqList) selectDicts = [] if (sysm.get_dfc_run_offline_status()): stat = "True" else: stat = "False" offline_mode = {"default": stat, "list": ["False", "True"]} selectDicts.append(offline_mode) get_select_defaults(dfmanager_input_form, dfmgr_offline_input_id, dfmgr_offline_input_idList, dfmgr_offline_input_typeList, selectDicts) dfmanager_input_form.set_shortForm(True) dfmanager_input_form.set_gridwidth(480) dfmanager_input_form.set_custombwidth(100) dfmanager_input_form.set_fullparms(True) dfmgr_input_html = dfmanager_input_form.get_html() dfmgr_heading_html = "<div>Run dfc Offline Manager</div>" gridclasses = ["dfcleanser-common-grid-header", "dfc-main"] gridhtmls = [dfmgr_heading_html, dfmgr_input_html] print("\n") if (cfg.get_dfc_mode() == cfg.INLINE_MODE): display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses, gridhtmls) else: display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses, gridhtmls, True) from dfcleanser.common.display_utils import display_pop_up_buffer display_pop_up_buffer()
def process_county_cities(parms): opstat = opStatus() fparms = get_parms_for_input(parms, suzw.county_cities_input_idList) state = fparms[0][:2] county = fparms[1] cfg.set_config_value(suzw.county_cities_input_id + "Parms", fparms) suzw.display_get_cities_for_county(parms) print("\n") citiesHeader = [""] citiesRows = [] citiesWidths = [20, 80] citiesAligns = ["left", "left"] primary_cities = suzm.get_cities_for_county(state, county, city_type=suzm.ANY_CITY_TYPE) if (not (primary_cities is None)): citiesRows.append(["US Zipcode Cities", str(primary_cities)]) cities_table = None from dfcleanser.common.table_widgets import dcTable, get_row_major_table, ROW_MAJOR, SCROLL_DOWN cities_table = dcTable( "Cities For " + str(county) + " - " + str(suzm.get_state_name(state).upper()), 'citiescodesid', cfg.SWZipcodeUtility_ID, citiesHeader, citiesRows, citiesWidths, citiesAligns) cities_table.set_small(True) cities_table.set_checkLength(False) cities_table.set_border(True) cities_table.set_tabletype(ROW_MAJOR) cities_table.set_rowspertable(50) citiesHtml = get_row_major_table(cities_table, SCROLL_DOWN, False) gridclasses = ["dfc-top"] gridhtmls = [citiesHtml] display_generic_grid("display-geocode-coords-wrapper", gridclasses, gridhtmls)
def display_get_cities_for_state(parms): state_cities_form = InputForm( state_cities_input_id, state_cities_input_idList, state_cities_input_labelList, state_cities_input_typeList, state_cities_input_placeholderList, state_cities_input_jsList, state_cities_input_reqList) selectDicts = [] from dfcleanser.sw_utilities.sw_utility_model import get_Dict states_dict = get_Dict("US_States_and_Territories") state_keys = list(states_dict.keys()) state_keys.sort() states_list = [] for i in range(len(state_keys)): states_list.append( str(state_keys[i]) + " : " + str(states_dict.get(state_keys[i]))) state_sel = {"default": states_list[0], "list": states_list} selectDicts.append(state_sel) get_select_defaults(state_cities_form, state_cities_input_id, state_cities_input_idList, state_cities_input_typeList, selectDicts) state_cities_form.set_gridwidth(720) state_cities_form.set_custombwidth(100) state_cities_form.set_fullparms(True) state_cities_input_html = "" state_cities_input_html = state_cities_form.get_html() state_cities_heading_html = "<div>Get State Cities</div>" gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"] gridhtmls = [state_cities_heading_html, state_cities_input_html] print("\n") display_generic_grid("geocode-utility-wrapper", gridclasses, gridhtmls)
def display_scripting_forms(): from dfcleanser.scripting.data_scripting_control import get_current_scriptlog if (not (get_current_scriptlog() == None)): cfg.set_config_value(dc_script_input_id + "Parms", [get_current_scriptlog()]) print("\n") notes = [] notes.append("You can manually edit the script log as desired") notes.append( "The step numbers are cmmentary only and order is insignificant") display_notes(notes) script_form = InputForm(dc_script_input_id, dc_script_input_idList, dc_script_input_labelList, dc_script_input_typeList, dc_script_input_placeholderList, dc_script_input_jsList, dc_script_input_reqList) script_form.set_shortForm(True) script_form.set_buttonstyle({ "font-size": 12, "height": 75, "width": 140, "left-margin": 70 }) script_form.set_gridwidth(880) script_form.set_fullparms(True) script_form_html = script_form.get_html() script_title_html = "<div>Scripting</div><br>" gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"] gridhtmls = [script_title_html, script_form_html] from dfcleanser.common.common_utils import display_generic_grid display_generic_grid("data-scripting-wrapper", gridclasses, gridhtmls) cfg.drop_config_value(dc_script_input_id + "Parms") print("\n")
def display_add_df_input(): """ * -------------------------------------------------------------------------- * function : add a dataframe to the dfc list * * parms : * * returns : N/A * -------------------------------------------------------- """ from dfcleanser.common.html_widgets import InputForm dfmanager_input_form = InputForm( dfmgr_add_input_id, dfmgr_add_input_idList, dfmgr_add_input_labelList, dfmgr_add_input_typeList, dfmgr_add_input_placeholderList, dfmgr_add_input_jsList, dfmgr_add_input_reqList) dfmanager_input_form.set_shortForm(True) dfmanager_input_form.set_gridwidth(480) dfmanager_input_form.set_custombwidth(70) dfmanager_input_form.set_fullparms(True) dfmgr_input_html = dfmanager_input_form.get_html() dfmgr_heading_html = "<div>Add df to dfc dataframe Manager</div>" gridclasses = ["dfcleanser-common-grid-header", "dfc-main"] gridhtmls = [dfmgr_heading_html, dfmgr_input_html] print("\n") if (cfg.get_dfc_mode() == cfg.INLINE_MODE): display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses, gridhtmls) else: display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses, gridhtmls, True) from dfcleanser.common.display_utils import display_pop_up_buffer display_pop_up_buffer()
def show_sys_info(): """ * -------------------------------------------------------------------------- * function : display dfcleanser system info * * parms : * N/A * * returns : N/A * -------------------------------------------------------- """ title = "Installed Python Info" ptitles = ["Version", "API", "Info"] pvalues = [ str(get_python_version()), str(sys.api_version), str(sys.version_info) ] parms_html = displayParms(title, ptitles, pvalues, cfg.System_ID, None, 0, False, 11) notes_html = show_setup_notes() libs_html = show_libs_info() libs_notes_html = show_libs_notes() gridclasses = ["dfc-header", "dfc-top", "dfc-bottom", "dfc-footer"] gridhtmls = [parms_html, notes_html, libs_html, libs_notes_html] if (cfg.get_dfc_mode() == cfg.INLINE_MODE): display_generic_grid("df-sys-info-wrapper", gridclasses, gridhtmls) else: display_generic_grid("df-sys-info-pop-up-wrapper", gridclasses, gridhtmls) from dfcleanser.common.display_utils import display_pop_up_buffer display_pop_up_buffer()
def display_system_chapters_taskbar(): """ * -------------------------------------------------------------------------- * function : display system taskbar * * parms : * N/A * * returns : N/A * -------------------------------------------------------- """ print("\n") dfc_utils_modules_checkbox = CheckboxGroupForm( dfc_utils_modules_checkbox_id, dfc_utils_modules_checkbox_idList, dfc_utils_modules_checkbox_labelList, dfc_utils_modules_checkbox_jsList, get_current_checkboxes(sysm.UTILITIES)) if (not (cfg.get_dfc_mode() == cfg.INLINE_MODE)): dfc_utils_modules_checkbox.set_rows_count([2, 2, 1]) dfc_utils_modules_heading_html = "<div>dfcleanser Utilities</div><br>" dfc_utils_modules_checkbox_html = dfc_utils_modules_checkbox.get_html() gridclasses = ["dfcleanser-common-grid-header", "dfc-main"] gridhtmls = [ dfc_utils_modules_heading_html, dfc_utils_modules_checkbox_html ] if (cfg.get_dfc_mode() == cfg.INLINE_MODE): display_generic_grid("dfcleanser-chapters-wrapper", gridclasses, gridhtmls) else: display_generic_grid("dfc-common-480px-2-vert-wrapper", gridclasses, gridhtmls, True) dfc_script_modules_tb = ButtonGroupForm(system_chapters_tb_doc_id, system_chapters_tb_keyTitleList, system_chapters_tb_jsList, system_chapters_tb_centered) dfc_script_modules_tb.set_gridwidth(480) dfc_script_modules_tb.set_custombwidth(160) dfc_script_modules_tb_html = dfc_script_modules_tb.get_html() gridclasses = ["dfc-footer"] gridhtmls = [dfc_script_modules_tb_html] display_generic_grid("dfcleanser-chapters-tb-wrapper", gridclasses, gridhtmls) print("\n")
def display_inspect_datatypes(option, df_data_info): """ * -------------------------------------------------------------------------- * function : display the datatypes option * * parms : * * returns : * N/A * -------------------------------------------------------- """ opstat = opStatus() import matplotlib.pyplot as plt clock = RunningClock() clock.start() try: if (not (option == dim.DISPLAY_FULL_COLUMN_NAMES)): data_types_table = dcTable("Column Data Types", "datatypesTable", cfg.DataInspection_ID) else: data_types_table = None data_types_html = diw.display_df_datatypes(data_types_table, df_data_info[0], df_data_info[1], df_data_info[2], option, False) gridclasses = ["dfc-main"] gridhtmls = [data_types_html] if (cfg.get_dfc_mode() == cfg.INLINE_MODE): display_generic_grid("df-inspection-wrapper", gridclasses, gridhtmls) else: display_generic_grid("df-inspection-pop-up-wrapper", gridclasses, gridhtmls) print("\n") import matplotlib.pyplot as plt import numpy as np font = {'fontsize': 14} font2 = {'fontsize': 18} objects = [] for i in range(len(df_data_info[0])): ttype = str(df_data_info[0][i]) ttype = ttype.replace("datetime.", "") objects.append(ttype) y_pos = np.arange(len(objects)) plt.bar(y_pos, df_data_info[1], align='center', alpha=0.5, color='#428bca') plt.xticks(y_pos, objects, rotation='vertical') plt.ylabel('Type Counts', fontdict=font) plt.xlabel('Data Types', fontdict=font) plt.title('Column Data Types', fontdict=font2) plt.show() except Exception as e: opstat.store_exception("Error displaying data types\n ", e) clock.stop() if (not (opstat.get_status())): display_exception(opstat)
def display_inspect_rows(rowid=0): """ * -------------------------------------------------------------------------- * function : display the inspect rows option * * parms : * * returns : * N/A * -------------------------------------------------------- """ opstat = opStatus() clock = RunningClock() clock.start() try: print("\n") from dfcleanser.data_transform.data_transform_dataframe_widgets import display_current_df_index display_current_df_index( cfg.get_current_chapter_df(cfg.DataInspection_ID), cfg.get_current_chapter_dfc_df_title(cfg.DataInspection_ID), 0, True) row_stats_html = diw.display_row_stats( cfg.get_current_chapter_df(cfg.DataInspection_ID), cfg.get_config_value(cfg.CURRENT_INSPECTION_DF), False) sample_row_html = dim.display_df_rows( cfg.get_current_chapter_df(cfg.DataInspection_ID), rowid, 200) rows_openexcel_tb = diw.get_inspection_openexcel_taskbar() rows_openexcel_tb.set_gridwidth(620) rows_openexcel_tb.set_customstyle({ "font-size": 13, "height": 90, "width": 120, "left-margin": 10 }) rows_openexcel_html = rows_openexcel_tb.get_html() rows_openexcel_html = (rows_openexcel_html + "<br>") cfg.set_config_value(cfg.CURRENT_SCROLL_ROW_KEY, rowid) gridclasses = ["dfc-top", "dfc-bottom", "dfc-footer"] gridhtmls = [row_stats_html, sample_row_html, rows_openexcel_html] if (cfg.get_dfc_mode() == cfg.INLINE_MODE): display_generic_grid("df-inspection-row-data-wrapper", gridclasses, gridhtmls) else: display_generic_grid("df-inspection-row-data-pop-up-wrapper", gridclasses, gridhtmls) except Exception as e: opstat.store_exception("Error displaying row data\n ", e) display_exception(opstat) import traceback traceback.print_exc() clock.stop()
def display_get_cities_for_county(parms): if (parms is None): fparms = cfg.get_config_value(county_cities_input_id + "Parms") if (not (fparms is None)): stateid = fparms[0] else: stateid = None else: fparms = get_parms_for_input(parms, county_cities_input_idList) stateid = fparms[0] cfg.set_config_value(county_cities_input_id + "Parms", fparms) county_cities_form = InputForm( county_cities_input_id, county_cities_input_idList, county_cities_input_labelList, county_cities_input_typeList, county_cities_input_placeholderList, county_cities_input_jsList, county_cities_input_reqList) selectDicts = [] from dfcleanser.sw_utilities.sw_utility_model import get_Dict states_dict = get_Dict("US_States_and_Territories") state_keys = list(states_dict.keys()) state_keys.sort() states_list = [] for i in range(len(state_keys)): states_list.append( str(state_keys[i]) + " : " + str(states_dict.get(state_keys[i]))) if (stateid is None): state_def = states_list[0] else: state_def = stateid state_sel = { "default": state_def, "list": states_list, "callback": "change_state_for_counties" } selectDicts.append(state_sel) state_id = state_def[:2] counties_list = suzm.get_counties_for_state(state_id) county_sel = {"default": counties_list[0], "list": counties_list} selectDicts.append(county_sel) get_select_defaults(county_cities_form, county_cities_input_id, county_cities_input_idList, county_cities_input_typeList, selectDicts) county_cities_form.set_gridwidth(720) county_cities_form.set_custombwidth(100) county_cities_form.set_fullparms(True) county_cities_input_html = "" county_cities_input_html = county_cities_form.get_html() county_cities_heading_html = "<div>Get County Cities</div>" gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"] gridhtmls = [county_cities_heading_html, county_cities_input_html] print("\n") display_generic_grid("geocode-utility-wrapper", gridclasses, gridhtmls)
def display_df_subset_setup(): """ * -------------------------------------------------------------------------- * function : display current df subset form * * parms : * df - dataframe to subset from * filters - filters form * colname - filters column name * * returns : N/A * -------------------------------------------------------- """ df_title = cfg.get_config_value(cfg.CURRENT_SUBSET_DF) df = cfg.get_dfc_dataframe_df(df_title) col_stats_table = get_column_stats_table(df_title, df) from dfcleanser.common.html_widgets import InputForm subset_input_form = InputForm(get_subset_input_id, get_subset_input_idList, get_subset_input_labelList, get_subset_input_typeList, get_subset_input_placeholderList, get_subset_input_jsList, get_subset_input_reqList) selectDicts = [] dataframes = cfg.get_dfc_dataframes_select_list(cfg.SWDFSubsetUtility_ID) selectDicts.append(dataframes) current_df = cfg.get_current_chapter_df(cfg.SWDFSubsetUtility_ID) colnames = current_df.columns.tolist() cols_name_list = [" "] for i in range(len(colnames)): cols_name_list.append(colnames[i]) cnames = { "default": cols_name_list[0], "list": cols_name_list, "callback": "change_subset_cols" } selectDicts.append(cnames) subssel = {"default": "Keep", "list": ["Keep", "Drop"]} selectDicts.append(subssel) get_select_defaults(subset_input_form, get_subset_input_form[0], get_subset_input_form[1], get_subset_input_form[3], selectDicts) subset_input_form.set_shortForm(False) subset_input_form.set_gridwidth(680) subset_input_form.set_custombwidth(140) subset_input_form.set_fullparms(True) get_subset_input_html = subset_input_form.get_html() get_subset_heading_html = "<div>Get Dataframe Subset</div><br></br>" gridclasses = ["dfc-top", "dfcleanser-common-grid-header", "dfc-bottom"] gridhtmls = [ col_stats_table, get_subset_heading_html, get_subset_input_html ] print("\n") display_generic_grid("sw-utils-subset-wrapper", gridclasses, gridhtmls)
def display_list_dict(stype, keyValue=None): """ * ------------------------------------------------------------------------ * function : display the dict or list start form * * parms : * id - list or dict * keyValue - dict or list name * * ------------------------------------------------------------------------- """ #print("display_list_dict",stype,keyValue) if (stype == swum.DICT_ID): list_dict_form = build_dict_utility_input_form else: list_dict_form = build_list_utility_input_form list_dict_input_form = InputForm(list_dict_form[0], list_dict_form[1], list_dict_form[2], list_dict_form[3], list_dict_form[4], list_dict_form[5], list_dict_form[6]) selectDicts = [] if (stype == swum.DICT_ID): from dfcleanser.sw_utilities.sw_utility_model import get_dicts_names, DFC_CREATED dict_names = get_dicts_names(DFC_CREATED) if (keyValue is None): def_dict = dict_names[0] else: def_dict = keyValue #print("def_dict",def_dict,keyValue) dictssel = { "default": def_dict, "list": dict_names, "callback": "select_dict" } seldict = swum.get_Dict(def_dict, DFC_CREATED) keys = list(seldict.keys()) if ((def_dict == "Country_Codes") or (def_dict == "Language_Codes")): keys.sort() seldict = swum.get_pretty_dict(seldict, keys) else: from dfcleanser.sw_utilities.sw_utility_model import get_lists_names, DFC_CREATED list_names = get_lists_names(DFC_CREATED) if (keyValue is None): def_list = list_names[0] else: def_list = keyValue dictssel = { "default": def_list, "list": list_names, "callback": "select_list" } sellist = str(swum.get_List(def_list, DFC_CREATED)) selectDicts.append(dictssel) from dfcleanser.common.common_utils import get_select_defaults get_select_defaults(list_dict_input_form, list_dict_form[0], list_dict_form[1], list_dict_form[3], selectDicts) list_dict_input_form.set_gridwidth(700) list_dict_input_form.set_custombwidth(120) list_dict_input_form.set_fullparms(True) if (stype == swum.DICT_ID): selparms = [def_dict, seldict] cfg.set_config_value(build_dict_utility_input_id + "Parms", selparms) cfg.set_config_value(build_dict_utility_input_id + "ParmsProtect", [False, True]) help_note = "To retrieve a dict in python from dfcleanser.sw_utiliities.sw_utility.model call 'get_Dict(dictname)'.</br>To add a dict in python from dfcleanser.sw_utiliities.sw_utility.model call 'add_Dict(dictname,newdict)'" else: selparms = [def_list, sellist] cfg.set_config_value(build_list_utility_input_id + "Parms", selparms) cfg.set_config_value(build_list_utility_input_id + "ParmsProtect", [False, True]) help_note = "To retrieve a list in python from dfcleanser.sw_utiliities.sw_utility.model call 'get_List(listname)'.</br>To add a list in python from dfcleanser.sw_utiliities.sw_utility.model call 'add_List(listtname,newlist)'" from dfcleanser.common.common_utils import get_help_note_html listdict_notes_html = get_help_note_html(help_note, 100, None, None) list_dictcustom_html = "" list_dictcustom_html = list_dict_input_form.get_html() if (stype == swum.DICT_ID): list_title_html = "<div>dfc Dicts</div><br></br>" cfg.drop_config_value(build_dict_utility_input_id + "Parms") else: list_title_html = "<div>dfc Lists</div><br></br>" cfg.drop_config_value(build_list_utility_input_id + "Parms") gridclasses = ["dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"] gridhtmls = [list_title_html, list_dictcustom_html, listdict_notes_html] print("\n") display_generic_grid("sw-utils-listdict-wrapper", gridclasses, gridhtmls)
def process_zipcode_attributes(parms): DEBUG_PROC_ZC_ATTRS = False opstat = opStatus() fparms = get_parms_for_input(parms, suzw.zipcode_atributes_input_idList) zipcode = fparms[0] cfg.set_config_value(suzw.zipcode_atributes_input_id + "Parms", fparms) suzw.display_get_zipcode_attributes(parms) print("\n") zipattrsHeader = [""] zipattrsRows = [] zipattrsWidths = [30, 70] zipattrsAligns = ["left", "left"] primary_cities = suzm.get_cities_for_zipcode( zipcode, city_type=suzm.PRIMARY_CITY_TYPE) acceptable_cities = suzm.get_cities_for_zipcode( zipcode, city_type=suzm.ACCEPTABLE_CITY_TYPE) not_acceptable_cities = suzm.get_cities_for_zipcode( zipcode, city_type=suzm.NOT_ACCEPTABLE_CITY_TYPE) zipcode_county = suzm.get_county_for_zipcode(zipcode) zipcode_state = suzm.get_state_for_zipcode(zipcode) zipcode_latitude = suzm.get_latitude_for_zipcode(zipcode) zipcode_longitude = suzm.get_longitude_for_zipcode(zipcode) zipcode_areacodes = suzm.get_areacodes_for_zipcode(zipcode) zipcode_active_status = suzm.is_zipcode_active(zipcode) zipcode_type = suzm.get_type_for_zipcode(zipcode) """ 501 603 604 8720 8732 8753 9001 9203 9204 9213 11708 """ if (DEBUG_PROC_ZC_ATTRS): print("primary_cities", primary_cities) if (not (acceptable_cities is None)): print("acceptable_cities", len(acceptable_cities), acceptable_cities) else: print("acceptable_cities", acceptable_cities) if (not (not_acceptable_cities is None)): print("not_acceptable_cities", len(not_acceptable_cities), not_acceptable_cities) else: print("not_acceptable_cities", not_acceptable_cities) print("zipcode_county", zipcode_county) print("zipcode_state", zipcode_state) print("zipcode_latitude", type(zipcode_latitude), zipcode_latitude) print("zipcode_longitude", type(zipcode_longitude), zipcode_longitude) print("zipcode_areacodes", type(zipcode_areacodes), zipcode_areacodes) print("zipcode_active_status", zipcode_active_status) print("zipcode_type", zipcode_type) if ((not (zipcode_active_status)) and (primary_cities is None)): zipattrsRows.append(["Current Status", "Zipcode is Invalid"]) else: if (zipcode_active_status): zipattrsRows.append(["Current Status", "Active"]) else: zipattrsRows.append(["Current Status", "Decommissioined"]) if (primary_cities is None): zipattrsRows.append(["Primary City", "None"]) else: if ((primary_cities == suzm.APO_ZIPCODE_TYPE) or (primary_cities == suzm.FPO_ZIPCODE_TYPE) or (primary_cities == suzm.DPO_ZIPCODE_TYPE)): zipattrsRows.append(["Primary City", "Washington DC"]) else: zipattrsRows.append(["Primary City", str(primary_cities)]) if (zipcode_county is None): zipattrsRows.append(["County", "None"]) else: zipattrsRows.append(["County", str(zipcode_county)]) if (zipcode_state is None): zipattrsRows.append(["State", "None"]) else: zipattrsRows.append(["State", str(zipcode_state)]) if ((zipcode_latitude is None) or (zipcode_longitude is None) or (numpy.isnan(zipcode_latitude)) or (numpy.isnan(zipcode_longitude))): zipattrsRows.append(["[Latitude,Longitude]", "Unknown"]) else: zipattrsRows.append([ "[Latitude,Longitude]", "[" + str(round(zipcode_latitude, 7)) + " , " + str(round(zipcode_longitude, 7)) + "]" ]) if (zipcode_type is None): zipattrsRows.append(["Zipcode Type", "None"]) else: if (zipcode_type == suzm.UNIQUE_ZIPCODE_TYPE): zipattrsRows.append(["Zipcode Type", str(suzm.UNIQUE_text)]) elif (zipcode_type == suzm.STANDARD_ZIPCODE_TYPE): zipattrsRows.append(["Zipcode Type", str(suzm.STANDARD_text)]) elif (zipcode_type == suzm.PO_BOX_ZIPCODE_TYPE): zipattrsRows.append(["Zipcode Type", str(suzm.PO_BOX_text)]) elif (zipcode_type == suzm.APO_ZIPCODE_TYPE): zipattrsRows.append(["Zipcode Type", str(suzm.APO_text)]) elif (zipcode_type == suzm.FPO_ZIPCODE_TYPE): zipattrsRows.append(["Zipcode Type", str(suzm.FPO_text)]) elif (zipcode_type == suzm.DPO_ZIPCODE_TYPE): zipattrsRows.append(["Zipcode Type", str(suzm.DPO_text)]) else: zipattrsRows.append(["Zipcode Type", "Unknown"]) if (not (zipcode_areacodes is None)): if (type(zipcode_areacodes) == list): for i in range(len(zipcode_areacodes)): zipcode_areacodes[i] = zipcode_areacodes[i].replace( "'", "") zipattrsRows.append(["Area Codes", str(zipcode_areacodes)]) if (not (acceptable_cities is None)): if (len(acceptable_cities) > 0): zipattrsRows.append( ["Acceptable Cities", str(acceptable_cities)]) if (not (not_acceptable_cities is None)): if (len(not_acceptable_cities) > 0): zipattrsRows.append( ["Not Acceptable Cities", str(not_acceptable_cities)]) zipattrs_table = None from dfcleanser.common.table_widgets import dcTable, get_row_major_table, ROW_MAJOR, SCROLL_DOWN zipattrs_table = dcTable("Zipcode " + str(zipcode) + " Properties", 'zipcodeattrsid', cfg.SWZipcodeUtility_ID, zipattrsHeader, zipattrsRows, zipattrsWidths, zipattrsAligns) zipattrs_table.set_small(True) zipattrs_table.set_checkLength(False) zipattrs_table.set_border(True) zipattrs_table.set_tabletype(ROW_MAJOR) zipattrs_table.set_rowspertable(50) zipattrsHtml = get_row_major_table(zipattrs_table, SCROLL_DOWN, False) gridclasses = ["dfc-top"] gridhtmls = [zipattrsHtml] display_generic_grid("display-geocode-coords-wrapper", gridclasses, gridhtmls)
def display_dict_maint(keyValue=None, loadfile=None): """ * ------------------------------------------------------------------------ * function : display the user dicts maintenance form * * parms : * keyValue - dict name * * ------------------------------------------------------------------------- """ #print("display_dict_maint",keyValue,loadfile) opstat = opStatus() if (loadfile is None): dict_maint_input_form = InputForm( maint_dict_utility_input_id, maint_dict_utility_input_idList, maint_dict_utility_input_labelList, maint_dict_utility_input_typeList, maint_dict_utility_input_placeholderList, maint_dict_utility_input_jsList, maint_dict_utility_input_reqList) else: dict_maint_input_form = InputForm( maint_dict_file_utility_input_id, maint_dict_file_utility_input_idList, maint_dict_file_utility_input_labelList, maint_dict_file_utility_input_typeList, maint_dict_file_utility_input_placeholderList, maint_dict_file_utility_input_jsList, maint_dict_file_utility_input_reqList) selectDicts = [] from dfcleanser.sw_utilities.sw_utility_model import get_dicts_names, USER_CREATED dict_names = get_dicts_names(USER_CREATED) #print("dict_names",dict_names) if (not (dict_names is None)): if (keyValue is None): def_dict = dict_names[0] else: def_dict = keyValue seldict = swum.get_Dict(def_dict, USER_CREATED) keys = list(seldict.keys()) if ((def_dict == "Country_Codes") or (def_dict == "Language_Codes")): keys.sort() seldict = swum.get_pretty_dict(seldict, keys) else: dict_names = ["No User dicts defined"] def_dict = "No User dicts defined" seldict = "User defined dict" dictssel = { "default": def_dict, "list": dict_names, "callback": "select_dict" } selectDicts.append(dictssel) from dfcleanser.common.common_utils import get_select_defaults get_select_defaults(dict_maint_input_form, maint_dict_utility_input_id, maint_dict_utility_input_idList, maint_dict_utility_input_typeList, selectDicts) dict_maint_input_form.set_gridwidth(700) #dict_maint_input_form.set_custombwidth(110) if (loadfile is None): dict_maint_input_form.set_buttonstyle({ "font-size": 13, "height": 75, "width": 90, "left-margin": 20 }) else: dict_maint_input_form.set_buttonstyle({ "font-size": 13, "height": 75, "width": 90, "left-margin": 205 }) dict_maint_input_form.set_fullparms(True) cfg.drop_config_value(maint_dict_utility_input_id + "Parms") cfg.drop_config_value(maint_dict_utility_input_id + "ParmsProtect") if (not (loadfile is None)): import json #from dfcleanser.common.common_utils import does_file_exist #print("does_file_exist",does_file_exist(loadfile)) try: with open(loadfile, 'r') as ds_file: ds = json.load(ds_file) ds_file.close() keys = list(ds.keys()) seldict = swum.get_pretty_dict(ds, keys) #print(seldict) except Exception as e: opstat.set_errorMsg("invalid user file to load " + loadfile) opstat.set_exception(e) if (opstat.get_status()): if (loadfile is None): cfg.set_config_value(maint_dict_utility_input_id + "Parms", [def_dict, "", seldict, ""]) else: cfg.set_config_value(maint_dict_utility_input_id + "Parms", [def_dict, "", seldict, loadfile]) cfg.set_config_value(maint_dict_utility_input_id + "ParmsProtect", [True, False, True, True]) help_note = "To add a user dict enter parms and values above and click on 'Add User Dict'.</br>To update the current dict change values and click on 'Update User Dict'" from dfcleanser.common.common_utils import get_help_note_html dict_maint_notes_html = get_help_note_html(help_note, 80, 75, None) dict_maint_html = "Fill in new user dict parms or update currently displayed user dict." dict_maint_html = dict_maint_input_form.get_html() dict_maint_title_html = "<div>User Dicts</div><br></br>" gridclasses = [ "dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer" ] gridhtmls = [ dict_maint_title_html, dict_maint_html, dict_maint_notes_html ] #print(dict_maint_html) #print(dict_maint_notes_html) print("\n") display_generic_grid("sw-utils-listdict-wrapper", gridclasses, gridhtmls) else: display_exception(opstat) add_error_to_log("[Get User Dict from File] " + loadfile + str(sys.exc_info()[0].__name__))
def display_dc_data_scripting(optionId, parms=None): from IPython.display import clear_output clear_output() from dfcleanser.common.cfg import check_if_dc_init if (not check_if_dc_init()): from dfcleanser.common.display_utils import display_dfcleanser_taskbar display_dfcleanser_taskbar( ButtonGroupForm(dc_script_tb_id, dc_script_tb_keyTitleList, dc_script_tb_jsList, dc_script_tb_centered), False) from dfcleanser.scripting.data_scripting_control import clear_data_scripting_data clear_data_scripting_data() return from dfcleanser.common.display_utils import display_dfcleanser_taskbar display_dfcleanser_taskbar( ButtonGroupForm(dc_script_tb_id, dc_script_tb_keyTitleList, dc_script_tb_jsList, dc_script_tb_centered), False) if (parms == None): from dfcleanser.scripting.data_scripting_control import clear_data_scripting_data clear_data_scripting_data() else: funcid = int(parms[0]) if (funcid == dsm.TURN_ON_SCRIPTING): from dfcleanser.scripting.data_scripting_control import set_scripting_status set_scripting_status(True) from dfcleanser.common.common_utils import display_status_note display_status_note("Scripting is turned on") elif (funcid == dsm.TURN_OFF_SCRIPTING): from dfcleanser.scripting.data_scripting_control import set_scripting_status set_scripting_status(False) from dfcleanser.common.common_utils import display_status_note display_status_note("Scripting is turned off") elif (funcid == dsm.SHOW_CURRENT_SCRIPT): display_scripting_forms() elif (funcid == dsm.ADD_TO_CURRENT_SCRIPT): script_form = InputForm( dc_add_code_input_id, dc_add_code_input_idList, dc_add_code_input_labelList, dc_add_code_input_typeList, dc_add_code_input_placeholderList, dc_add_code_input_jsList, dc_add_code_input_reqList) script_form.set_shortForm(True) script_form.set_buttonstyle({ "font-size": 12, "height": 75, "width": 140, "left-margin": 70 }) script_form.set_gridwidth(880) script_form.set_fullparms(True) script_form_html = script_form.get_html() script_title_html = "<div>Scripting</div><br>" gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"] gridhtmls = [script_title_html, script_form_html] from dfcleanser.common.common_utils import display_generic_grid display_generic_grid("data-scripting-wrapper", gridclasses, gridhtmls) elif (funcid == dsm.ADD_CODE_SCRIPT): from dfcleanser.scripting.data_scripting_control import add_code_to_script add_code_to_script(parms) display_status("Code added to Current Script succesfully ") elif (funcid == dsm.DELETE_CURRENT_SCRIPT): from dfcleanser.scripting.data_scripting_control import drop_current_script drop_current_script() display_status("Current Script Cleared succesfully ") elif (funcid == dsm.LOAD_BACKUP_SCRIPT): from dfcleanser.scripting.data_scripting_control import load_backup_scriptlog_to_current load_backup_scriptlog_to_current() display_scripting_forms() display_status("Current Script Loaded from Backup ") elif (funcid == dsm.SAVE_BACKUP_SCRIPT): codeparms = get_parms_for_input(parms[1], dc_script_input_idList) save_code = get_code_from_form(codeparms) from dfcleanser.scripting.data_scripting_control import ( set_current_scriptlog, save_current_scriptlog_to_backup) set_current_scriptlog(save_code) save_current_scriptlog_to_backup(save_code) display_scripting_forms() display_status("Current Script Backed up successfully ") elif (funcid == dsm.RUN_CURRENT_SCRIPT): opstat = opStatus() from dfcleanser.scripting.data_scripting_control import run_scriptlog run_scriptlog(parms, opstat) if (opstat.get_status()): display_status("Current Script Run successfully ") else: display_exception(opstat)
def display_inspect_cols(parms): """ * -------------------------------------------------------------------------- * function : display the inspect cols option * * parms : * * returns : * N/A * -------------------------------------------------------- """ opstat = opStatus() clock = RunningClock() clock.start() try: df = cfg.get_current_chapter_df(cfg.DataInspection_ID) colnames = df.columns.tolist() if (not (parms is None)): colname = parms else: colname = colnames[0] cnames = { 'default': colname, 'list': colnames, "callback": "change_inspect_cols_col", "size": 10 } if (is_numeric_col(df, colname)): coldetails_form = InputForm(diw.inspect_col_input_id, diw.inspect_col_input_idList, diw.inspect_col_input_labelList, diw.inspect_col_input_typeList, diw.inspect_col_input_placeholderList, diw.inspect_col_input_jsList, diw.inspect_col_input_reqList) else: coldetails_form = InputForm( diw.inspect_nn_col_input_id, diw.inspect_nn_col_input_idList, diw.inspect_nn_col_input_labelList, diw.inspect_nn_col_input_typeList, diw.inspect_nn_col_input_placeholderList, diw.inspect_nn_col_input_jsList, diw.inspect_nn_col_input_reqList) selectDicts = [] selectDicts.append(cnames) get_select_defaults(coldetails_form, diw.inspect_col_input_id, diw.inspect_col_input_idList, diw.inspect_col_input_typeList, selectDicts) coldetails_form.set_shortForm(True) coldetails_form.set_fullparms(True) if (cfg.get_dfc_mode() == cfg.INLINE_MODE): coldetails_form.set_gridwidth(360) if (is_numeric_col(df, colname)): coldetails_form.set_buttonstyle({ "font-size": 12, "height": 75, "width": 85, "left-margin": 2 }) else: coldetails_form.set_buttonstyle({ "font-size": 12, "height": 75, "width": 85, "left-margin": 75 }) else: coldetails_form.set_gridwidth(480) if (is_numeric_col(df, colname)): coldetails_form.set_buttonstyle({ "font-size": 12, "height": 75, "width": 110, "left-margin": 2 }) else: coldetails_form.set_buttonstyle({ "font-size": 12, "height": 75, "width": 110, "left-margin": 110 }) coldetails_html = coldetails_form.get_html() from dfcleanser.data_cleansing.data_cleansing_widgets import display_col_stats col_stats_html = display_col_stats(df, colname, False, True) gridclasses = ["dfc-left", "dfc-right"] gridhtmls = [col_stats_html, coldetails_html] if (cfg.get_dfc_mode() == cfg.INLINE_MODE): display_generic_grid("df-inspection-column-data-wrapper", gridclasses, gridhtmls) else: display_generic_grid("df-inspection-pop-up-column-data-wrapper", gridclasses, gridhtmls) except Exception as e: opstat.store_exception("Error displaying column data\n ", e) clock.stop() if (not (opstat.get_status())): display_exception(opstat)
def display_process_subset(): """ * -------------------------------------------------------------------------- * function : display current df subset form * * parms : * df - dataframe to subset from * * returns : N/A * -------------------------------------------------------- """ current_step = swsm.get_current_subset_step() dftitle = current_step.get_output_subset_df_title() subset_df = swsm.get_current_subset_df() df_stats_table = get_df_stats_table(dftitle, subset_df) df_cols_table = get_column_stats_table(dftitle, subset_df) from dfcleanser.common.html_widgets import InputForm subset_input_form = InputForm( get_subset_run_input_id, get_subset_run_input_idList, get_subset_run_input_labelList, get_subset_run_input_typeList, get_subset_run_input_placeholderList, get_subset_run_input_jsList, get_subset_run_input_reqList) selectDicts = [] colnames = subset_df.columns.tolist() cols_name_list = [" "] for i in range(len(colnames)): cols_name_list.append(colnames[i]) cnames = { "default": cols_name_list[0], "list": cols_name_list, "callback": "change_run_subset_cols" } selectDicts.append(cnames) subssel = {"default": "Keep", "list": ["Keep", "Drop"]} selectDicts.append(subssel) savesel = {"default": "True", "list": ["True", "False"]} selectDicts.append(savesel) get_select_defaults(subset_input_form, get_subset_run_input_form[0], get_subset_run_input_form[1], get_subset_run_input_form[3], selectDicts) subset_input_form.set_shortForm(False) subset_input_form.set_gridwidth(680) subset_input_form.set_custombwidth(90) subset_input_form.set_fullparms(True) cfg.set_config_value(get_subset_run_input_id + "Parms", [dftitle, "", "", ""]) cfg.set_config_value(get_subset_run_input_id + "ParmsProtect", [True, False, False, False]) get_subset_input_html = subset_input_form.get_html() get_subset_heading_html = "<div>Process Dataframe Subset</div><br>" gridclasses = [ "dfc-top", "dfc-main", "dfcleanser-common-grid-header", "dfc-bottom" ] gridhtmls = [ df_stats_table, df_cols_table, get_subset_heading_html, get_subset_input_html ] print("\n") display_generic_grid("sw-utils-subset-run-wrapper", gridclasses, gridhtmls)
def process_zipcode_cities(parms): opstat = opStatus() fparms = get_parms_for_input(parms, suzw.zipcode_cities_input_idList) city = fparms[0] state = fparms[1][:2] cfg.set_config_value(suzw.zipcode_cities_input_id + "Parms", fparms) suzw.display_get_zips_for_city(parms) print("\n") cityzipsHeader = [""] cityzipsRows = [] cityzipsWidths = [30, 70] cityzipsAligns = ["left", "left"] cityzips = suzm.get_zipcodes_for_city( state, city, zipcode_type=suzm.STANDARD_ZIPCODE_TYPE, active_status=suzm.ACTIVE_STATUS_TYPE) citypobzips = suzm.get_zipcodes_for_city( state, city, zipcode_type=suzm.PO_BOX_ZIPCODE_TYPE, active_status=suzm.ACTIVE_STATUS_TYPE) cityuniquezips = suzm.get_zipcodes_for_city( state, city, zipcode_type=suzm.UNIQUE_ZIPCODE_TYPE, active_status=suzm.ACTIVE_STATUS_TYPE) citydecomzips = suzm.get_zipcodes_for_city( state, city, zipcode_type=suzm.ANY_ZIPCODE_TYPE, active_status=suzm.DECOMMISIONED_STATUS_TYPE) if (not (cityzips is None)): cityzipsRows.append([suzm.STANDARD_text + " Zipcodes", str(cityzips)]) if (not (citypobzips is None)): cityzipsRows.append([suzm.PO_BOX_text + " Zipcodes", str(citypobzips)]) if (not (cityuniquezips is None)): cityzipsRows.append( [suzm.UNIQUE_text + " Zipcodes", str(cityuniquezips)]) if (not (citydecomzips is None)): cityzipsRows.append( ["Decommissioned" + " Zipcodes", str(citydecomzips)]) cityzips_table = None from dfcleanser.common.table_widgets import dcTable, get_row_major_table, ROW_MAJOR, SCROLL_DOWN cityzips_table = dcTable( "Zipcodes For " + str(city.upper()) + ", " + str(state), 'cityzipcodesid', cfg.SWZipcodeUtility_ID, cityzipsHeader, cityzipsRows, cityzipsWidths, cityzipsAligns) cityzips_table.set_small(True) cityzips_table.set_checkLength(False) cityzips_table.set_border(True) cityzips_table.set_tabletype(ROW_MAJOR) cityzips_table.set_rowspertable(50) cityzipsHtml = get_row_major_table(cityzips_table, SCROLL_DOWN, False) gridclasses = ["dfc-top"] gridhtmls = [cityzipsHtml] display_generic_grid("display-geocode-coords-wrapper", gridclasses, gridhtmls)
def display_funcs(): """ * ------------------------------------------------------------------------ * function : display the dfc funcs start form * * parms : * * ------------------------------------------------------------------------- """ list_funcs_input_form = InputForm( build_funcs_utility_input_id, build_funcs_utility_input_idList, build_funcs_utility_input_labelList, build_funcs_utility_input_typeList, build_funcs_utility_input_placeholderList, build_funcs_utility_input_jsList, build_funcs_utility_input_reqList) selectDicts = [] from dfcleanser.sw_utilities.sw_utility_genfunc_model import get_genfunc_list gen_funcs = get_genfunc_list() default_func = gen_funcs[0] funcssel = { "default": default_func, "list": gen_funcs, "callback": "select_gen_func" } selectDicts.append(funcssel) from dfcleanser.common.common_utils import get_select_defaults get_select_defaults(list_funcs_input_form, build_funcs_utility_input_id, build_funcs_utility_input_idList, build_funcs_utility_input_typeList, selectDicts) list_funcs_input_form.set_gridwidth(700) list_funcs_input_form.set_custombwidth(120) list_funcs_input_form.set_fullparms(True) from dfcleanser.sw_utilities.sw_utility_genfunc_model import reservedfunctionsmodule from dfcleanser.sw_utilities.sw_utility_genfunc_functions import get_generic_function_code func_code = get_generic_function_code(default_func) cfg.set_config_value(build_funcs_utility_input_id + "Parms", [default_func, reservedfunctionsmodule, func_code]) cfg.set_config_value(build_funcs_utility_input_id + "ParmsProtect", [False, True, True]) help_note = "Generic functions help note'" from dfcleanser.common.common_utils import get_help_note_html list_funcs_notes_html = get_help_note_html(help_note, 100, None, None) list_funcs_html = "" list_funcs_html = list_funcs_input_form.get_html() list_funcs_title_html = "<div>Generic dfc Functions</div><br></br>" gridclasses = ["dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"] gridhtmls = [list_funcs_title_html, list_funcs_html, list_funcs_notes_html] print("\n") display_generic_grid("sw-utils-listdict-wrapper", gridclasses, gridhtmls)
def display_manual_df_subset_setup(sequence_title, input_df_title, colnames_list, cols_action, stepid): """ * -------------------------------------------------------------------------- * function : display current manual df subset form * * parms : * df - dataframe to subset from * filters - filters form * colname - filters column name * * returns : N/A * -------------------------------------------------------- """ df = swsm.get_current_subset_df() col_stats_table = get_column_stats_table(input_df_title, df) from dfcleanser.common.html_widgets import InputForm subset_input_form = InputForm(get_manual_input_id, get_manual_input_idList, get_manual_input_labelList, get_manual_input_typeList, get_manual_input_placeholderList, get_manual_input_jsList, get_manual_input_reqList) selectDicts = [] dataframes = { "default": input_df_title, "list": [input_df_title] } #cfg.get_dfc_dataframes_select_list(cfg.SWDFSubsetUtility_ID) selectDicts.append(dataframes) current_df = df #cfg.get_dfc_dataframe_df(input_df_title) colnames = current_df.columns.tolist() cols_name_list = [" "] for i in range(len(colnames)): cols_name_list.append(colnames[i]) cnames = { "default": cols_name_list[0], "list": cols_name_list, "callback": "change_subset_cols" } selectDicts.append(cnames) if (cols_action == "Keep"): subssel = {"default": "Keep", "list": ["Keep", "Drop"]} else: subssel = {"default": "Drop", "list": ["Keep", "Drop"]} selectDicts.append(subssel) get_select_defaults(subset_input_form, get_manual_input_form[0], get_manual_input_form[1], get_manual_input_form[3], selectDicts) if (len(colnames_list) > 0): cnames = str(colnames_list) else: cnames = "" cfg.set_config_value(get_manual_input_id + "Parms", ["", cnames, "", cols_action]) cfg.set_config_value(get_manual_input_id + "ParmsProtect", [True, False, False, False]) subset_input_form.set_shortForm(False) subset_input_form.set_gridwidth(680) subset_input_form.set_custombwidth(140) subset_input_form.set_fullparms(True) get_subset_input_html = subset_input_form.get_html() get_subset_heading_html = "<div>Run Subset Sequence '" + str( sequence_title) + "' Step " + str(stepid) + "</div><br></br>" gridclasses = ["dfc-top", "dfcleanser-common-grid-header", "dfc-bottom"] gridhtmls = [ col_stats_table, get_subset_heading_html, get_subset_input_html ] print("\n") display_generic_grid("sw-utils-subset-wrapper", gridclasses, gridhtmls)
def display_list_maint(keyValue=None, loadfile=None): """ * ------------------------------------------------------------------------ * function : display the user lists maintenance form * * parms : * keyValue - list name * * ------------------------------------------------------------------------- """ opstat = opStatus() if (loadfile is None): list_maint_input_form = InputForm( maint_list_utility_input_id, maint_list_utility_input_idList, maint_list_utility_input_labelList, maint_list_utility_input_typeList, maint_list_utility_input_placeholderList, maint_list_utility_input_jsList, maint_list_utility_input_reqList) else: list_maint_input_form = InputForm( maint_list_file_utility_input_id, maint_list_file_utility_input_idList, maint_list_file_utility_input_labelList, maint_list_file_utility_input_typeList, maint_list_file_utility_input_placeholderList, maint_list_file_utility_input_jsList, maint_list_file_utility_input_reqList) selectDicts = [] from dfcleanser.sw_utilities.sw_utility_model import get_lists_names, USER_CREATED list_names = get_lists_names(USER_CREATED) #print("list_names",list_names) if (not (list_names is None)): if (keyValue is None): def_list = list_names[0] else: def_list = keyValue sellist = swum.get_List(def_list, USER_CREATED) dsstr = "[" for i in range(len(sellist)): dsstr = dsstr + str(sellist[i]) if (i == (len(sellist) - 1)): dsstr = dsstr + "]" else: dsstr = dsstr + "," else: list_names = ["No User lists defined"] def_list = "No User lists defined" sellist = "User defined list" listssel = { "default": def_list, "list": list_names, "callback": "select_list" } selectDicts.append(listssel) from dfcleanser.common.common_utils import get_select_defaults get_select_defaults(list_maint_input_form, maint_list_utility_input_id, maint_list_utility_input_idList, maint_list_utility_input_typeList, selectDicts) list_maint_input_form.set_gridwidth(700) if (loadfile is None): list_maint_input_form.set_buttonstyle({ "font-size": 13, "height": 75, "width": 90, "left-margin": 20 }) else: list_maint_input_form.set_buttonstyle({ "font-size": 13, "height": 75, "width": 90, "left-margin": 205 }) list_maint_input_form.set_fullparms(True) cfg.drop_config_value(maint_list_utility_input_id + "Parms") cfg.drop_config_value(maint_list_utility_input_id + "ParmsProtect") if (not (loadfile is None)): import json from dfcleanser.common.common_utils import does_file_exist if (does_file_exist(loadfile)): try: with open(loadfile, 'r') as ds_file: ds = json.load(ds_file) ds_file.close() dsstr = "[" for i in range(len(ds)): dsstr = dsstr + str(ds[i]) if (i == (len(ds) - 1)): dsstr = dsstr + "]" else: dsstr = dsstr + "," except Exception as e: opstat.set_status(False) opstat.set_errorMsg("Error processing user file to load" + loadfile) opstat.set_exception(e) else: opstat.set_status(False) opstat.set_errorMsg("invalid user file to load" + loadfile) if (opstat.get_status()): if (loadfile is None): cfg.set_config_value(maint_list_utility_input_id + "Parms", [def_list, "", dsstr, ""]) else: cfg.set_config_value(maint_list_utility_input_id + "Parms", [def_list, "", dsstr, loadfile]) cfg.set_config_value(maint_list_utility_input_id + "ParmsProtect", [True, False, True, True]) help_note = "To add a user list enter parms and values above and click on 'Add User List'.</br>To update the current list change values and click on 'Update User List'" from dfcleanser.common.common_utils import get_help_note_html list_maint_notes_html = get_help_note_html(help_note, 80, 75, None) list_maint_html = "" list_maint_html = list_maint_input_form.get_html() list_maint_title_html = "<div>User Lists</div><br></br>" gridclasses = [ "dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer" ] gridhtmls = [ list_maint_title_html, list_maint_html, list_maint_notes_html ] #print(list_maint_html) #print(list_maint_notes_html) print("\n") display_generic_grid("sw-utils-listdict-wrapper", gridclasses, gridhtmls) else: display_exception(opstat) add_error_to_log("[Get User Dict from File] " + loadfile + str(sys.exc_info()[0].__name__))
def display_dc_pandas_export_sql_inputs(fId, dbId, dbconparms, exportparms=None): """ * -------------------------------------------------------------------------- * function : display pandas sql export form * * parms : * fid - export type * dbid - database id * dbconparms - db connector parms * exportparms - export parms * * returns : N/A * -------------------------------------------------------- """ opstat = opStatus() opstatStatus = True listHtml = "" dbid = int(dbId) fid = int(fId) fparms = None if (fid == 0): dbid = cfg.get_config_value(cfg.CURRENT_DB_ID_KEY) from dfcleanser.data_import.data_import_widgets import ( get_table_names, TABLE_NAMES, get_rows_html) tablelist = get_table_names(dbid, opstat) listHtml = get_rows_html(tablelist, TABLE_NAMES, True) elif (fid == 1): fparms = get_parms_for_input(exportparms, pandas_export_sqltable_idList) dbid = cfg.get_config_value(cfg.CURRENT_DB_ID_KEY) cfg.set_config_value(pandas_export_sqltable_id + "Parms", fparms) if (len(fparms[0]) > 0): from dfcleanser.data_import.data_import_widgets import ( get_column_names, get_rows_html) from dfcleanser.data_import.data_import_model import COLUMN_NAMES columnlist = get_column_names(dbid, fparms[1], opstat) listHtml = get_rows_html(columnlist, COLUMN_NAMES, True) else: opstat.set_status(False) opstat.set_errorMsg("No Table Selected") elif (fid == 2): cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbid) dbcondict = {} if (not (dbconparms == None)): parse_connector_parms(dbconparms, dbid, dbcondict) else: conparms = get_stored_con_Parms(dbid) parse_connector_parms(conparms, dbid, dbcondict) listHtml = get_db_connector_list(dbid, dbcondict) if (not (opstat.get_status())): dbcondict = {} conparms = get_stored_con_Parms(dbid) parse_connector_parms(conparms, dbid, dbcondict) listHtml = get_db_connector_list(dbid, dbcondict) opstatStatus = opstat.get_status() opstatErrormsg = opstat.get_errorMsg() opstat.set_status(True) if (opstat.get_status()): export_sql_input_form = InputForm( pandas_export_sqltable_id, pandas_export_sqltable_idList, pandas_export_sqltable_labelList, pandas_export_sqltable_typeList, pandas_export_sqltable_placeholderList, pandas_export_sqltable_jsList, pandas_export_sqltable_reqList) selectDicts = [] df_list = cfg.get_dfc_dataframes_select_list(cfg.DataExport_ID) selectDicts.append(df_list) exists = {"default": "fail", "list": ["fail", "replace", "append"]} selectDicts.append(exists) index = {"default": "True", "list": ["True", "False"]} selectDicts.append(index) get_select_defaults(export_sql_input_form, pandas_export_sqltable_id, pandas_export_sqltable_idList, pandas_export_sqltable_typeList, selectDicts) export_sql_input_form.set_shortForm(False) export_sql_input_form.set_gridwidth(680) export_sql_input_form.set_custombwidth(125) export_sql_input_form.set_fullparms(True) export_sql_input_html = "" export_sql_input_html = export_sql_input_form.get_html() export_sql_heading_html = "<div>" + get_pandas_export_input_title( dem.SQLTABLE_EXPORT, dbid) + "</div><br>" if (not (exportparms == None)): cfg.set_config_value(pandas_export_sqltable_id + "Parms", fparms) gridclasses = [ "dfcleanser-common-grid-header", "dfc-left", "dfc-right" ] gridhtmls = [export_sql_heading_html, listHtml, export_sql_input_html] display_generic_grid("data-import-sql-table-wrapper", gridclasses, gridhtmls) if (not (opstatStatus)): opstat.set_status(opstatStatus) opstat.set_errorMsg(opstatErrormsg) display_exception(opstat)