def display_df_criteria(df_title, df): """ * -------------------------------------------------------------------------- * function : display current df subset form * * parms : * df - dataframe to subset from * filters - filters form * colname - filters column name * * returns : N/A * -------------------------------------------------------- """ col_stats_table = get_column_stats_table(df_title, df) from dfcleanser.common.html_widgets import InputForm subset_input_form = InputForm(get_subset_criteria_input_id, get_subset_criteria_input_idList, get_subset_criteria_input_labelList, get_subset_criteria_input_typeList, get_subset_criteria_input_placeholderList, get_subset_criteria_input_jsList, get_subset_criteria_input_reqList) subset_input_form.set_label_note({ 2: [ dfchelp.SUBSET_CRITERIA, "dataframe subset criteria", "subsetcriteriaimageid" ] }) subset_input_form.set_textarea_resize_flag(1, False) subset_input_form.set_textarea_resize_flag(3, False) subset_input_form.set_shortForm(False) subset_input_form.set_gridwidth(620) subset_input_form.set_custombwidth(110) subset_input_form.set_fullparms(True) subset_input_form.set_custom_font_size("gsselectstring", 14) cfg.set_config_value(get_subset_criteria_input_id + "Parms", [ "", swsm.starting_criteria_preamble, swsm.starting_criteria, swsm.starting_criteria_postamble ]) cfg.set_config_value(get_subset_criteria_input_id + "ParmsProtect", [False, True, False, True]) get_subset_input_html = subset_input_form.get_html() get_subset_heading_html = "<div>Get Dataframe Subset</div><br></br>" gridclasses = ["dfc-top", "dfcleanser-common-grid-header", "dfc-bottom"] gridhtmls = [ col_stats_table, get_subset_heading_html, get_subset_input_html ] print("\n") display_generic_grid("sw-utils-subset-wrapper", gridclasses, gridhtmls)
def get_current_subset_df(parms) : fparms = get_parms_for_input(parms[0],["dsdfdataframe"]) if(len(fparms) > 0) : selected_df = fparms[0] if(not (len(selected_df) == 0) ) : cfg.set_config_value(cfg.CURRENT_SUBSET_DF,selected_df)
def display_sequence_save_subset(df_title, df, auto=False): """ * -------------------------------------------------------------------------- * function : display current df subset run * * parms : * df - dataframe to subset from * * returns : N/A * -------------------------------------------------------- """ df_stats_table = get_df_stats_table(df_title, df) df_cols_table = get_column_stats_table(df_title, df) from dfcleanser.common.html_widgets import InputForm subset_input_form = InputForm( get_saved_save_input_id, get_saved_save_input_idList, get_saved_save_input_labelList, get_saved_save_input_typeList, get_saved_save_input_placeholderList, get_saved_save_input_jsList, get_saved_save_input_reqList) selectDicts = [] dropsel = {"default": "False", "list": ["True", "False"]} selectDicts.append(dropsel) get_select_defaults(subset_input_form, get_saved_save_input_id, get_saved_save_input_idList, get_saved_save_input_typeList, selectDicts) subset_input_form.set_shortForm(False) subset_input_form.set_gridwidth(680) subset_input_form.set_custombwidth(140) subset_input_form.set_fullparms(True) cfg.set_config_value( get_saved_save_input_id + "Parms", [df_title, "./" + df_title + ".csv", df_title, "False"]) cfg.drop_config_value(get_saved_save_input_id + "ParmsProtect") get_subset_input_html = subset_input_form.get_html() get_subset_heading_html = "<div>Save Dataframe Subset</div><br>" gridclasses = [ "dfc-top", "dfc-main", "dfcleanser-common-grid-header", "dfc-bottom" ] gridhtmls = [ df_stats_table, df_cols_table, get_subset_heading_html, get_subset_input_html ] print("\n") display_generic_grid("sw-utils-subset-run-wrapper", gridclasses, gridhtmls)
def process_county_cities(parms): opstat = opStatus() fparms = get_parms_for_input(parms, suzw.county_cities_input_idList) state = fparms[0][:2] county = fparms[1] cfg.set_config_value(suzw.county_cities_input_id + "Parms", fparms) suzw.display_get_cities_for_county(parms) print("\n") citiesHeader = [""] citiesRows = [] citiesWidths = [20, 80] citiesAligns = ["left", "left"] primary_cities = suzm.get_cities_for_county(state, county, city_type=suzm.ANY_CITY_TYPE) if (not (primary_cities is None)): citiesRows.append(["US Zipcode Cities", str(primary_cities)]) cities_table = None from dfcleanser.common.table_widgets import dcTable, get_row_major_table, ROW_MAJOR, SCROLL_DOWN cities_table = dcTable( "Cities For " + str(county) + " - " + str(suzm.get_state_name(state).upper()), 'citiescodesid', cfg.SWZipcodeUtility_ID, citiesHeader, citiesRows, citiesWidths, citiesAligns) cities_table.set_small(True) cities_table.set_checkLength(False) cities_table.set_border(True) cities_table.set_tabletype(ROW_MAJOR) cities_table.set_rowspertable(50) citiesHtml = get_row_major_table(cities_table, SCROLL_DOWN, False) gridclasses = ["dfc-top"] gridhtmls = [citiesHtml] display_generic_grid("display-geocode-coords-wrapper", gridclasses, gridhtmls)
def display_scripting_forms(): from dfcleanser.scripting.data_scripting_control import get_current_scriptlog if (not (get_current_scriptlog() == None)): cfg.set_config_value(dc_script_input_id + "Parms", [get_current_scriptlog()]) print("\n") notes = [] notes.append("You can manually edit the script log as desired") notes.append( "The step numbers are cmmentary only and order is insignificant") display_notes(notes) script_form = InputForm(dc_script_input_id, dc_script_input_idList, dc_script_input_labelList, dc_script_input_typeList, dc_script_input_placeholderList, dc_script_input_jsList, dc_script_input_reqList) script_form.set_shortForm(True) script_form.set_buttonstyle({ "font-size": 12, "height": 75, "width": 140, "left-margin": 70 }) script_form.set_gridwidth(880) script_form.set_fullparms(True) script_form_html = script_form.get_html() script_title_html = "<div>Scripting</div><br>" gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"] gridhtmls = [script_title_html, script_form_html] from dfcleanser.common.common_utils import display_generic_grid display_generic_grid("data-scripting-wrapper", gridclasses, gridhtmls) cfg.drop_config_value(dc_script_input_id + "Parms") print("\n")
def process_custom_export(fparms, exportId, display=True): """ * -------------------------------------------------------------------------- * function : custom export * * parms : * fparms - export parms * exportId - export id * display - display flag * * returns : N/A * -------------------------------------------------------- """ opstat = opStatus() fparms[0] = fparms[0].replace("\n", "<br/>") try: exec(fparms[0]) except Exception as e: opstat.store_exception("Unable to export custom", e) if (opstat.get_status()): if (display): #make scriptable script = [ "# Export Custom ", "from dfcleanser.data_export.data_export_control import process_custom_export", "process_custom_export(" + json.dumps(fparms) + "," + str(exportId) + ",False)" ] add_to_script(script, opstat) if (len(fparms) > 0): cfg.set_config_value(exportId + "Parms", "custom") cfg.set_config_value(cfg.CURRENT_EXPORTED_FILE_NAME_KEY, "custom", True) return (opstat)
def export_custom(parms): """ * -------------------------------------------------------------------------- * function : export a custom * * parms : * parms - sql parms * * returns : N/A * -------------------------------------------------------- """ functionid = parms[0] opstat = opStatus() dispstats = False if (functionid == 1): opstat = process_custom_export(parms[1], dew.custom_export_id, display=True) dispstats = True elif (functionid == 2): custom_code = "# custom export\n" custom_code = custom_code + parms[1] cfg.set_config_value(dew.custom_export_id + "Parms", custom_code) display_export_forms(dem.EXPORT_CUSTOM_ONLY) elif (functionid == 3): cfg.drop_config_value(dew.custom_export_id + "Parms") display_export_forms(dem.EXPORT_CUSTOM_ONLY) elif (functionid == 5): display_export_forms(dem.EXPORT_CUSTOM_ONLY, -1, True) return (dispstats, opstat)
def process_zipcode_cities(parms): opstat = opStatus() fparms = get_parms_for_input(parms, suzw.zipcode_cities_input_idList) city = fparms[0] state = fparms[1][:2] cfg.set_config_value(suzw.zipcode_cities_input_id + "Parms", fparms) suzw.display_get_zips_for_city(parms) print("\n") cityzipsHeader = [""] cityzipsRows = [] cityzipsWidths = [30, 70] cityzipsAligns = ["left", "left"] cityzips = suzm.get_zipcodes_for_city( state, city, zipcode_type=suzm.STANDARD_ZIPCODE_TYPE, active_status=suzm.ACTIVE_STATUS_TYPE) citypobzips = suzm.get_zipcodes_for_city( state, city, zipcode_type=suzm.PO_BOX_ZIPCODE_TYPE, active_status=suzm.ACTIVE_STATUS_TYPE) cityuniquezips = suzm.get_zipcodes_for_city( state, city, zipcode_type=suzm.UNIQUE_ZIPCODE_TYPE, active_status=suzm.ACTIVE_STATUS_TYPE) citydecomzips = suzm.get_zipcodes_for_city( state, city, zipcode_type=suzm.ANY_ZIPCODE_TYPE, active_status=suzm.DECOMMISIONED_STATUS_TYPE) if (not (cityzips is None)): cityzipsRows.append([suzm.STANDARD_text + " Zipcodes", str(cityzips)]) if (not (citypobzips is None)): cityzipsRows.append([suzm.PO_BOX_text + " Zipcodes", str(citypobzips)]) if (not (cityuniquezips is None)): cityzipsRows.append( [suzm.UNIQUE_text + " Zipcodes", str(cityuniquezips)]) if (not (citydecomzips is None)): cityzipsRows.append( ["Decommissioned" + " Zipcodes", str(citydecomzips)]) cityzips_table = None from dfcleanser.common.table_widgets import dcTable, get_row_major_table, ROW_MAJOR, SCROLL_DOWN cityzips_table = dcTable( "Zipcodes For " + str(city.upper()) + ", " + str(state), 'cityzipcodesid', cfg.SWZipcodeUtility_ID, cityzipsHeader, cityzipsRows, cityzipsWidths, cityzipsAligns) cityzips_table.set_small(True) cityzips_table.set_checkLength(False) cityzips_table.set_border(True) cityzips_table.set_tabletype(ROW_MAJOR) cityzips_table.set_rowspertable(50) cityzipsHtml = get_row_major_table(cityzips_table, SCROLL_DOWN, False) gridclasses = ["dfc-top"] gridhtmls = [cityzipsHtml] display_generic_grid("display-geocode-coords-wrapper", gridclasses, gridhtmls)
def display_manual_df_subset_setup(sequence_title, input_df_title, colnames_list, cols_action, stepid): """ * -------------------------------------------------------------------------- * function : display current manual df subset form * * parms : * df - dataframe to subset from * filters - filters form * colname - filters column name * * returns : N/A * -------------------------------------------------------- """ df = swsm.get_current_subset_df() col_stats_table = get_column_stats_table(input_df_title, df) from dfcleanser.common.html_widgets import InputForm subset_input_form = InputForm(get_manual_input_id, get_manual_input_idList, get_manual_input_labelList, get_manual_input_typeList, get_manual_input_placeholderList, get_manual_input_jsList, get_manual_input_reqList) selectDicts = [] dataframes = { "default": input_df_title, "list": [input_df_title] } #cfg.get_dfc_dataframes_select_list(cfg.SWDFSubsetUtility_ID) selectDicts.append(dataframes) current_df = df #cfg.get_dfc_dataframe_df(input_df_title) colnames = current_df.columns.tolist() cols_name_list = [" "] for i in range(len(colnames)): cols_name_list.append(colnames[i]) cnames = { "default": cols_name_list[0], "list": cols_name_list, "callback": "change_subset_cols" } selectDicts.append(cnames) if (cols_action == "Keep"): subssel = {"default": "Keep", "list": ["Keep", "Drop"]} else: subssel = {"default": "Drop", "list": ["Keep", "Drop"]} selectDicts.append(subssel) get_select_defaults(subset_input_form, get_manual_input_form[0], get_manual_input_form[1], get_manual_input_form[3], selectDicts) if (len(colnames_list) > 0): cnames = str(colnames_list) else: cnames = "" cfg.set_config_value(get_manual_input_id + "Parms", ["", cnames, "", cols_action]) cfg.set_config_value(get_manual_input_id + "ParmsProtect", [True, False, False, False]) subset_input_form.set_shortForm(False) subset_input_form.set_gridwidth(680) subset_input_form.set_custombwidth(140) subset_input_form.set_fullparms(True) get_subset_input_html = subset_input_form.get_html() get_subset_heading_html = "<div>Run Subset Sequence '" + str( sequence_title) + "' Step " + str(stepid) + "</div><br></br>" gridclasses = ["dfc-top", "dfcleanser-common-grid-header", "dfc-bottom"] gridhtmls = [ col_stats_table, get_subset_heading_html, get_subset_input_html ] print("\n") display_generic_grid("sw-utils-subset-wrapper", gridclasses, gridhtmls)
def display_process_subset(): """ * -------------------------------------------------------------------------- * function : display current df subset form * * parms : * df - dataframe to subset from * * returns : N/A * -------------------------------------------------------- """ current_step = swsm.get_current_subset_step() dftitle = current_step.get_output_subset_df_title() subset_df = swsm.get_current_subset_df() df_stats_table = get_df_stats_table(dftitle, subset_df) df_cols_table = get_column_stats_table(dftitle, subset_df) from dfcleanser.common.html_widgets import InputForm subset_input_form = InputForm( get_subset_run_input_id, get_subset_run_input_idList, get_subset_run_input_labelList, get_subset_run_input_typeList, get_subset_run_input_placeholderList, get_subset_run_input_jsList, get_subset_run_input_reqList) selectDicts = [] colnames = subset_df.columns.tolist() cols_name_list = [" "] for i in range(len(colnames)): cols_name_list.append(colnames[i]) cnames = { "default": cols_name_list[0], "list": cols_name_list, "callback": "change_run_subset_cols" } selectDicts.append(cnames) subssel = {"default": "Keep", "list": ["Keep", "Drop"]} selectDicts.append(subssel) savesel = {"default": "True", "list": ["True", "False"]} selectDicts.append(savesel) get_select_defaults(subset_input_form, get_subset_run_input_form[0], get_subset_run_input_form[1], get_subset_run_input_form[3], selectDicts) subset_input_form.set_shortForm(False) subset_input_form.set_gridwidth(680) subset_input_form.set_custombwidth(90) subset_input_form.set_fullparms(True) cfg.set_config_value(get_subset_run_input_id + "Parms", [dftitle, "", "", ""]) cfg.set_config_value(get_subset_run_input_id + "ParmsProtect", [True, False, False, False]) get_subset_input_html = subset_input_form.get_html() get_subset_heading_html = "<div>Process Dataframe Subset</div><br>" gridclasses = [ "dfc-top", "dfc-main", "dfcleanser-common-grid-header", "dfc-bottom" ] gridhtmls = [ df_stats_table, df_cols_table, get_subset_heading_html, get_subset_input_html ] print("\n") display_generic_grid("sw-utils-subset-run-wrapper", gridclasses, gridhtmls)
def display_dc_pandas_export_sql_inputs(fId, dbId, dbconparms, exportparms=None): """ * -------------------------------------------------------------------------- * function : display pandas sql export form * * parms : * fid - export type * dbid - database id * dbconparms - db connector parms * exportparms - export parms * * returns : N/A * -------------------------------------------------------- """ opstat = opStatus() opstatStatus = True listHtml = "" dbid = int(dbId) fid = int(fId) fparms = None if (fid == 0): dbid = cfg.get_config_value(cfg.CURRENT_DB_ID_KEY) from dfcleanser.data_import.data_import_widgets import ( get_table_names, TABLE_NAMES, get_rows_html) tablelist = get_table_names(dbid, opstat) listHtml = get_rows_html(tablelist, TABLE_NAMES, True) elif (fid == 1): fparms = get_parms_for_input(exportparms, pandas_export_sqltable_idList) dbid = cfg.get_config_value(cfg.CURRENT_DB_ID_KEY) cfg.set_config_value(pandas_export_sqltable_id + "Parms", fparms) if (len(fparms[0]) > 0): from dfcleanser.data_import.data_import_widgets import ( get_column_names, get_rows_html) from dfcleanser.data_import.data_import_model import COLUMN_NAMES columnlist = get_column_names(dbid, fparms[1], opstat) listHtml = get_rows_html(columnlist, COLUMN_NAMES, True) else: opstat.set_status(False) opstat.set_errorMsg("No Table Selected") elif (fid == 2): cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbid) dbcondict = {} if (not (dbconparms == None)): parse_connector_parms(dbconparms, dbid, dbcondict) else: conparms = get_stored_con_Parms(dbid) parse_connector_parms(conparms, dbid, dbcondict) listHtml = get_db_connector_list(dbid, dbcondict) if (not (opstat.get_status())): dbcondict = {} conparms = get_stored_con_Parms(dbid) parse_connector_parms(conparms, dbid, dbcondict) listHtml = get_db_connector_list(dbid, dbcondict) opstatStatus = opstat.get_status() opstatErrormsg = opstat.get_errorMsg() opstat.set_status(True) if (opstat.get_status()): export_sql_input_form = InputForm( pandas_export_sqltable_id, pandas_export_sqltable_idList, pandas_export_sqltable_labelList, pandas_export_sqltable_typeList, pandas_export_sqltable_placeholderList, pandas_export_sqltable_jsList, pandas_export_sqltable_reqList) selectDicts = [] df_list = cfg.get_dfc_dataframes_select_list(cfg.DataExport_ID) selectDicts.append(df_list) exists = {"default": "fail", "list": ["fail", "replace", "append"]} selectDicts.append(exists) index = {"default": "True", "list": ["True", "False"]} selectDicts.append(index) get_select_defaults(export_sql_input_form, pandas_export_sqltable_id, pandas_export_sqltable_idList, pandas_export_sqltable_typeList, selectDicts) export_sql_input_form.set_shortForm(False) export_sql_input_form.set_gridwidth(680) export_sql_input_form.set_custombwidth(125) export_sql_input_form.set_fullparms(True) export_sql_input_html = "" export_sql_input_html = export_sql_input_form.get_html() export_sql_heading_html = "<div>" + get_pandas_export_input_title( dem.SQLTABLE_EXPORT, dbid) + "</div><br>" if (not (exportparms == None)): cfg.set_config_value(pandas_export_sqltable_id + "Parms", fparms) gridclasses = [ "dfcleanser-common-grid-header", "dfc-left", "dfc-right" ] gridhtmls = [export_sql_heading_html, listHtml, export_sql_input_html] display_generic_grid("data-import-sql-table-wrapper", gridclasses, gridhtmls) if (not (opstatStatus)): opstat.set_status(opstatStatus) opstat.set_errorMsg(opstatErrormsg) display_exception(opstat)
def display_list_dict(stype, keyValue=None): """ * ------------------------------------------------------------------------ * function : display the dict or list start form * * parms : * id - list or dict * keyValue - dict or list name * * ------------------------------------------------------------------------- """ #print("display_list_dict",stype,keyValue) if (stype == swum.DICT_ID): list_dict_form = build_dict_utility_input_form else: list_dict_form = build_list_utility_input_form list_dict_input_form = InputForm(list_dict_form[0], list_dict_form[1], list_dict_form[2], list_dict_form[3], list_dict_form[4], list_dict_form[5], list_dict_form[6]) selectDicts = [] if (stype == swum.DICT_ID): from dfcleanser.sw_utilities.sw_utility_model import get_dicts_names, DFC_CREATED dict_names = get_dicts_names(DFC_CREATED) if (keyValue is None): def_dict = dict_names[0] else: def_dict = keyValue #print("def_dict",def_dict,keyValue) dictssel = { "default": def_dict, "list": dict_names, "callback": "select_dict" } seldict = swum.get_Dict(def_dict, DFC_CREATED) keys = list(seldict.keys()) if ((def_dict == "Country_Codes") or (def_dict == "Language_Codes")): keys.sort() seldict = swum.get_pretty_dict(seldict, keys) else: from dfcleanser.sw_utilities.sw_utility_model import get_lists_names, DFC_CREATED list_names = get_lists_names(DFC_CREATED) if (keyValue is None): def_list = list_names[0] else: def_list = keyValue dictssel = { "default": def_list, "list": list_names, "callback": "select_list" } sellist = str(swum.get_List(def_list, DFC_CREATED)) selectDicts.append(dictssel) from dfcleanser.common.common_utils import get_select_defaults get_select_defaults(list_dict_input_form, list_dict_form[0], list_dict_form[1], list_dict_form[3], selectDicts) list_dict_input_form.set_gridwidth(700) list_dict_input_form.set_custombwidth(120) list_dict_input_form.set_fullparms(True) if (stype == swum.DICT_ID): selparms = [def_dict, seldict] cfg.set_config_value(build_dict_utility_input_id + "Parms", selparms) cfg.set_config_value(build_dict_utility_input_id + "ParmsProtect", [False, True]) help_note = "To retrieve a dict in python from dfcleanser.sw_utiliities.sw_utility.model call 'get_Dict(dictname)'.</br>To add a dict in python from dfcleanser.sw_utiliities.sw_utility.model call 'add_Dict(dictname,newdict)'" else: selparms = [def_list, sellist] cfg.set_config_value(build_list_utility_input_id + "Parms", selparms) cfg.set_config_value(build_list_utility_input_id + "ParmsProtect", [False, True]) help_note = "To retrieve a list in python from dfcleanser.sw_utiliities.sw_utility.model call 'get_List(listname)'.</br>To add a list in python from dfcleanser.sw_utiliities.sw_utility.model call 'add_List(listtname,newlist)'" from dfcleanser.common.common_utils import get_help_note_html listdict_notes_html = get_help_note_html(help_note, 100, None, None) list_dictcustom_html = "" list_dictcustom_html = list_dict_input_form.get_html() if (stype == swum.DICT_ID): list_title_html = "<div>dfc Dicts</div><br></br>" cfg.drop_config_value(build_dict_utility_input_id + "Parms") else: list_title_html = "<div>dfc Lists</div><br></br>" cfg.drop_config_value(build_list_utility_input_id + "Parms") gridclasses = ["dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"] gridhtmls = [list_title_html, list_dictcustom_html, listdict_notes_html] print("\n") display_generic_grid("sw-utils-listdict-wrapper", gridclasses, gridhtmls)
def display_data_inspection(option, parms=None): """ * -------------------------------------------------------------------------- * function : main data inspection processing * * parms : * option - function option * parms - associated parms * * returns : * N/A * -------------------------------------------------------- """ from IPython.display import clear_output clear_output() opstat = opStatus() from dfcleanser.common.html_widgets import define_inputs, are_owner_inputs_defined if (not (are_owner_inputs_defined(cfg.DataInspection_ID))): define_inputs(cfg.DataInspection_ID, diw.datainspection_inputs) if (option == dim.MAIN_OPTION): drop_working_df() diw.display_dfc_inspection_main() clear_data_inspection_data() else: diw.display_inspection_main_taskbar() if (cfg.is_a_dfc_dataframe_loaded()): if ((option == dim.DISPLAY_DATATYPES_OPTION) or (option == dim.DISPLAY_NANS_OPTION) or (option == dim.DISPLAY_ROWS_OPTION) or (option == dim.DISPLAY_COLS_OPTION) or (option == dim.DISPLAY_CATEGORIES_OPTION)): fparms = get_parms_for_input(parms[0], diw.data_inspection_df_input_idList) if (len(fparms) > 0): cfg.set_config_value(cfg.CURRENT_INSPECTION_DF, fparms[0]) if (not (option == dim.DISPLAY_ROWS_OPTION)): drop_working_df() if ((option == dim.DISPLAY_DATATYPES_OPTION) or (option == dim.DISPLAY_FULL_COLUMN_NAMES)): df_data_info = dim.get_df_datatypes_data( cfg.get_current_chapter_df(cfg.DataInspection_ID)) display_inspect_datatypes(option, df_data_info) elif (option == dim.DISPLAY_NANS_OPTION): display_inspect_nans() elif (option == dim.DISPLAY_ROWS_OPTION): display_inspect_rows() elif (option == dim.DISPLAY_COLS_OPTION): if (len(parms) > 1): display_inspect_cols(parms[1]) else: display_inspect_cols(None) elif (option == dim.DISPLAY_CATEGORIES_OPTION): display_inspect_categories() elif ((option == dim.DROP_ROW_NANS_OPTION) or (option == dim.DROP_COL_NANS_OPTION)): thresholdType = parms[0] if (option == dim.DROP_ROW_NANS_OPTION): fparms = get_parms_for_input(parms[1], diw.drop_rows_input_idList) else: fparms = get_parms_for_input(parms[1], diw.drop_columns_input_idList) if (len(fparms) > 0): try: threshold = int(fparms[0]) except: opstat.set_status(False) if (option == dim.DROP_ROW_NANS_OPTION): opstat.set_errorMsg("Drop Nan Rows Threshold value '" + fparms[0] + "' is invalid") else: opstat.set_errorMsg("Drop Nan Cols Threshold value '" + fparms[0] + "' is invalid") threshold = None else: opstat.set_status(False) if (option == dim.DROP_ROW_NANS_OPTION): opstat.set_errorMsg( "Drop Nan Rows Threshold value is not defined") else: opstat.set_errorMsg( "Drop Nan Cols Threshold value is not defined") threshold = None if (option == dim.DROP_ROW_NANS_OPTION): if (opstat.get_status()): dropstats = drop_nan_rows( cfg.get_current_chapter_df(cfg.DataInspection_ID), threshold, thresholdType, opstat) if (not (opstat.get_status())): display_exception(opstat) else: if (dropstats[0] > 0): display_status( str(dropstats[0]) + " Nan Rows Dropped Successfully") else: display_status( "No Rows matching threshold were dropped") else: if (opstat.get_status()): numcolsdropped = drop_nan_cols( cfg.get_current_chapter_df(cfg.DataInspection_ID), threshold, thresholdType, opstat) if (not (opstat.get_status())): display_exception(opstat) else: if (numcolsdropped > 0): display_status( str(numcolsdropped) + " Columns with Nans Dropped Successfully") else: display_status( " No Columns matching threshold were dropped") elif (option == dim.DISPLAY_ROW_OPTION): display_inspect_rows() elif (option == dim.DISPLAY_COL_GRAPHS): display_inspect_graphs(parms) elif (option == dim.DISPLAY_COL_OUTLIERS): display_inspect_outliers(parms[0]) elif (option == dim.DISPLAY_SCROLL_TO_DF_ROW): diw.display_scroll_to_row() elif (option == dim.PROCESS_SCROLL_TO_DF_ROW): opstat = opStatus() df = cfg.get_current_chapter_df(cfg.DataInspection_ID) retparms = get_row_id_for_df(df, parms, diw.scroll_df_rows_input_idList, opstat) if (opstat.get_status()): if (retparms[1] == 0): display_inspect_rows(retparms[0]) else: display_inspect_rows(retparms[0]) else: diw.display_scroll_to_row() display_exception(opstat) elif (option == dim.SCROLL_DF_ROWS_DOWN): new_row_id = cfg.get_config_value(cfg.CURRENT_SCROLL_ROW_KEY) if (new_row_id is None): new_row_id = 0 else: new_row_id = new_row_id + 200 df = cfg.get_current_chapter_df(cfg.DataInspection_ID) if (new_row_id > len(df)): new_row_id = cfg.get_config_value( cfg.CURRENT_SCROLL_ROW_KEY) display_inspect_rows(new_row_id) elif (option == dim.SCROLL_DF_ROWS_UP): new_row_id = cfg.get_config_value(cfg.CURRENT_SCROLL_ROW_KEY) if (new_row_id is None): new_row_id = 0 else: new_row_id = new_row_id - 200 if (new_row_id < 0): new_row_id = 0 display_inspect_rows(new_row_id) elif (option == dim.DISPLAY_DF_ROW): print("dim.DISPLAY_DF_ROW") elif (option == dim.DISPLAY_DF_ROW_REMOTE): chapterid = parms[0] #print("chapterId",chapterid) new_config_df = None if (chapterid == cfg.DataInspection_ID): new_config_df = cfg.get_config_value(cfg.CURRENT_INSPECTION_DF) elif (chapterid == cfg.DataCleansing_ID): new_config_df = cfg.get_config_value(cfg.CURRENT_CLEANSE_DF) elif (chapterid == cfg.DataTransform_ID): new_config_df = cfg.get_config_value(cfg.CURRENT_TRANSFORM_DF) elif (chapterid == cfg.DataExport_ID): new_config_df = cfg.get_config_value(cfg.CURRENT_EXPORT_DF) elif (chapterid == cfg.DataImport_ID): new_config_df = cfg.get_config_value(cfg.CURRENT_IMPORT_DF) elif (chapterid == cfg.SWGeocodeUtility_ID): new_config_df = cfg.get_config_value(cfg.CURRENT_GEOCODE_DF) elif (chapterid == cfg.SWDFSubsetUtility_ID): new_config_df = cfg.get_config_value(cfg.CURRENT_SUBSET_DF) cfg.set_config_value(cfg.CURRENT_INSPECTION_DF, new_config_df) display_inspect_rows() else: cfg.drop_config_value(cfg.CURRENT_INSPECTION_DF) if (not (option == dim.MAIN_OPTION)): cfg.display_no_dfs(cfg.DataInspection_ID) from dfcleanser.common.display_utils import display_pop_up_buffer display_pop_up_buffer()
def display_system_environment(funcId, parms=None): """ * -------------------------------------------------------------------------- * function : display system environment screens * * parms : * funcId - display func id * parms - associated parms * * returns : * N/A * -------------------------------------------------------- """ if (not (cfg.check_if_dc_init())): sysw.display_system_main_taskbar() return else: from dfcleanser.common.html_widgets import define_inputs, are_owner_inputs_defined if (not (are_owner_inputs_defined(cfg.System_ID))): define_inputs(cfg.System_ID, sysw.system_inputs) if (funcId == sysm.DISPLAY_MAIN): display_main_tb() clear_system_data() if (not (funcId == sysm.PROCESS_EULA)): if (not isEULA_read()): #display_system_main_taskbar() sysw.display_EULA() return if (funcId == sysm.DISPLAY_CHAPTERS): sysw.display_system_chapters_taskbar() if (funcId == sysm.RESET_CHAPTERS): sysw.display_system_main_taskbar() initialize_notebook() if (funcId == sysm.PROCESS_CHAPTERS): parms[0] = parms[0].replace("[", "") parms[0] = parms[0].replace("]", "") utils_cbs = parms[0].split(",") utilscbs = [] for i in range(len(utils_cbs)): if (utils_cbs[i] == '"True"'): utilscbs.append(1) else: utilscbs.append(0) from dfcleanser.system.load import reload_dfcleanser reload_dfcleanser([utilscbs]) clear_cell() sysw.display_system_main_taskbar() elif (funcId == sysm.DISPLAY_DATAFRAMES): if (not (parms is None)): title = parms[0] else: title = None sysw.display_system_main_taskbar() sysw.display_df_dataframes(title) elif (funcId == sysm.DISPLAY_ADD_DATAFRAME): sysw.display_system_main_taskbar() cfg.drop_config_value(sysw.dfmgr_add_input_id + "Parms") sysw.display_add_df_input() elif (funcId == sysm.PROCESS_DATAFRAME): fid = parms[0] from dfcleanser.common.common_utils import get_parms_for_input fparms = get_parms_for_input(parms[1], sysw.dfmgr_input_idList) dftitle = None if (fid == sysm.DROP_DATAFRAME): cfg.drop_dfc_dataframe(fparms[0]) elif (fid == sysm.SET_DATAFRAME): print("sysm.SET_DATAFRAME") elif (fid == sysm.UPDATE_DATAFRAME): cfg.set_dfc_dataframe_notes(fparms[0], fparms[3]) dftitle = fparms[0] elif (fid == sysm.RENAME_DATAFRAME): cfg.rename_dfc_dataframe( cfg.get_config_value(cfg.CURRENT_DF_DISPLAYED_KEY), fparms[0]) dftitle = fparms[0] sysw.display_system_main_taskbar() sysw.display_df_dataframes(dftitle) elif (funcId == sysm.PROCESS_ADD_DATAFRAME): opstat = opStatus() from dfcleanser.common.common_utils import get_parms_for_input fparms = get_parms_for_input(parms[1], sysw.dfmgr_add_input_idList) dftitle = fparms[0] dfobject = fparms[1] dfnotes = fparms[2] if (not (len(dftitle)) > 0): opstat.set_status(False) opstat.set_errorMsg("Invalid df title parm") else: if (len(dfobject) > 0): try: add_df_js = ("add_new_dfc_df('" + dftitle + "', '" + dfobject + "', '" + dfnotes + "');") run_jscript(add_df_js, "fail to add dataframe : ") except Exception: opstat.set_status(False) opstat.set_errorMsg( "Unable to add df to dfc manager : " + str(sys.exc_info()[0].__name__)) else: opstat.set_status(False) opstat.set_errorMsg("Invalid df name parm") if (opstat.get_status()): sysw.display_df_dataframes(dftitle) else: display_status(opstat.get_errorMsg()) sysw.display_add_df_input() elif (funcId == sysm.DISPLAY_SYSTEM): display_main_tb() sysw.show_sys_info() elif (funcId == sysm.DISPLAY_OFFLINE): display_main_tb() sysw.display_offline() elif (funcId == sysm.DISPLAY_ABOUT): sysw.display_system_main_taskbar() sysw.show_about_info() elif (funcId == sysm.DISPLAY_DFC_FILES): sysw.display_system_main_taskbar() sysw.display_dfc_files_form() elif (funcId == sysm.DISPLAY_EULA): display_main_tb() sysw.display_EULA() elif (funcId == sysm.DISPLAY_README): display_main_tb() sysw.display_README() elif (funcId == sysm.PROCESS_EULA): display_main_tb() cfg.set_config_value(cfg.EULA_FLAG_KEY, "true") elif (funcId == sysm.EXIT_SETUP): from dfcleanser.system.load import unload_dfcleanser unload_dfcleanser() return
def export_pandas_html(fparms, exportId, labellist, display=True): """ * -------------------------------------------------------------------------- * function : pandas html export * * parms : * fparms - export parms * exportId - export id * labellist - parm label list * display - display flag * * returns : N/A * -------------------------------------------------------- """ opstat = opStatus() if (len(fparms) == 0): opstat.set_status(False) opstat.set_errorMsg("No Export parameters defined") else: try: htmlkeys = [labellist[2], labellist[3], labellist[4], labellist[5]] htmlvals = [fparms[2], fparms[3], fparms[4], fparms[5]] htmltypes = [INT_PARM, BOOLEAN_PARM, BOOLEAN_PARM, STRING_PARM] htmlparms = {} htmladdlparms = {} except Exception as e: opstat.store_exception("Error parsing import parms", e) if (opstat.get_status()): try: htmlparms = get_function_parms(htmlkeys, htmlvals, htmltypes) if (not (fparms[6] == "")): htmladdlparms = json.loads(fparms[6]) if (len(htmladdlparms) > 0): addlparmskeys = htmladdlparms.keys() for i in range(len(addlparmskeys)): htmlparms.update({ addlparmskeys[i]: htmladdlparms.get(addlparmskeys[i]) }) except Exception as e: opstat.store_exception("Unable to get additional parms", e) if (opstat.get_status()): if (fparms[0] == ""): opstat.set_status(False) opstat.set_errorMsg("No dataframe slected") else: df = cfg.get_dfc_dataframe_df(fparms[0]) try: if (len(htmlparms) > 0): df.to_html(fparms[1], **htmlparms) else: df.to_html(fparms[1]) except Exception as e: opstat.store_exception( "Unable to export html file" + fparms[0], e) if (opstat.get_status()): if (display): #make scriptable script = [ "# Export HTML File ", "dfcleanser.data_export.data_export_control import export_pandas_html", "export_pandas_html(" + json.dumps(fparms) + "," + str(exportId) + "," + json.dumps(labellist) + ",False)" ] add_to_script(script, opstat) if (len(fparms) > 0): cfg.set_config_value(exportId + "Parms", fparms) cfg.set_config_value(cfg.CURRENT_EXPORTED_FILE_NAME_KEY, fparms[0], True) return (opstat)
def save_data_export_start(): cfg.set_config_value(cfg.CURRENT_EXPORT_START_TIME, time.time())
def display_dfsubset_utility(optionId,parms=None) : """ * --------------------------------------------------------- * function : main subset utility control * * parms : * optionId - function to run * parms - parms to ryn function * * returns : * NA * -------------------------------------------------------- """ if(cfg.is_a_dfc_dataframe_loaded()) : from IPython.display import clear_output clear_output() from dfcleanser.common.html_widgets import define_inputs, are_owner_inputs_defined if(not (are_owner_inputs_defined(cfg.SWDFSubsetUtility_ID)) ) : define_inputs(cfg.SWDFSubsetUtility_ID,swsw.SWUtility_subset_inputs) if(optionId == swsm.DISPLAY_MAIN) : swsw.get_dfsubset_main_taskbar() clear_sw_utility_dfsubsetdata() cfg.display_data_select_df(cfg.SWDFSubsetUtility_ID) swsm.clear_current_subset_data() elif(optionId == swsm.DISPLAY_GET_SUBSET) : swsm.clear_current_subset_data() if(DEBUG_SUBSET) : swsm.set_current_subset_sequence(swsm.dfc_subset_sequence()) print("\ncurrent_subset_sequence\n") print("input_df_title",swsm.get_current_subset_sequence().get_input_df_title()) print("get_sequence_title",swsm.get_current_subset_sequence().get_sequence_title()) print("get_sequence_steps",swsm.get_current_subset_sequence().get_sequence_steps()) if(not (swsm.get_current_subset_sequence().get_sequence_steps() is None)) : print("get_total_sequence_steps",swsm.get_current_subset_sequence().get_total_sequence_steps()) print("get_output_csv",swsm.get_current_subset_sequence().get_output_csv()) print("get_output_dfc_df_title",swsm.get_current_subset_sequence().get_output_dfc_df_title()) swsw.display_df_subset_setup() if(DEBUG_SUBSET) : print("DISPLAY_GET_SUBSET",parms) print("DISPLAY_GET_SUBSET : clear data") print(swsm.get_current_subset_sequence()) print(swsm.get_current_subset_df()) print(swsm.get_current_subset_step()) print("new_sequence") swsm.dump_current_step() swsm.dump_current_sequence() elif(optionId == swsm.PROCESS_GET_SUBSET) : current_step = swsm.get_current_subset_step() if(not (current_step is None)) : current_sequence = swsm.get_current_subset_sequence() current_sequence.add_step_to_sequence_steps(current_step) fparms = get_parms_for_input(parms,swsw.get_subset_input_idList) if(len(fparms) > 0) : df_title = fparms[0] df = cfg.get_dfc_dataframe_df(df_title) col_names = fparms[1] col_action = fparms[3] new_subset_df = drop_add_cols(col_names,col_action,df) new_subset_df_title = df_title new_subset_step = swsm.dfc_subset_step(new_subset_df_title,col_names,col_action) swsm.set_current_subset_step(new_subset_step) swsm.set_current_subset_df(new_subset_df) swsw.display_df_criteria(new_subset_df_title,new_subset_df) if(DEBUG_SUBSET) : print("\nPROCESS_GET_SUBSET\n ",parms,"\n ",fparms) swsm.dump_current_step() swsm.dump_current_sequence() elif(optionId == swsm.DISPLAY_SAVED_SUBSET) : swsw.display_saved_subset_sequences() if(DEBUG_SUBSET) : print("\nDISPLAY_SAVED_SUBSET",parms) elif(optionId == swsm.PROCESS_RUN_CRITERIA) : opstat = opStatus() fparms = get_parms_for_input(parms,swsw.get_subset_criteria_input_idList) subset_title = fparms[0] if(len(subset_title) == 0) : current_sequence = swsm.get_current_subset_sequence() total_steps = current_sequence.get_total_sequence_steps() current_step = swsm.get_current_subset_step() subset_title = current_step.get_input_subset_df_title() + "_subset_" + str(total_steps+1) criteria = fparms[2] if(len(criteria) > 0) : try : clock = RunningClock() clock.start() final_criteria = (swsm.starting_criteria_preamble + criteria + swsm.starting_criteria_postamble) exec(final_criteria) current_step = swsm.get_current_subset_step() current_step.set_criteria(criteria) current_step.set_output_subset_df_title(subset_title) clock.stop() except Exception as e: opstat.store_exception("Error running df_criteria " + criteria,e) clock.stop() if(opstat.get_status()) : swsw.display_process_subset() else : display_exception(opstat) if(DEBUG_SUBSET) : print("PROCESS_RUN_CRITERIA : End") swsm.dump_current_step() swsm.dump_current_sequence() elif(optionId == swsm.DISPLAY_SAVE_SUBSET) : fparms = get_parms_for_input(parms,swsw.get_subset_run_input_idList) current_sequence = swsm.get_current_subset_sequence() current_step = swsm.get_current_subset_step() current_sequence.add_step_to_sequence_steps(current_step) if(len(fparms) > 0) : df_title = fparms[0] df = swsm.get_current_subset_df() col_names = fparms[1] col_action = fparms[3] new_subset_df = drop_add_cols(col_names,col_action,df) swsw.display_save_subset(df_title,new_subset_df) if(DEBUG_SUBSET) : print("DISPLAY_SAVE_SUBSET",parms,fparms) swsm.dump_current_step() swsm.dump_current_sequence() elif(optionId == swsm.DISPLAY_SAVE_AND_GET_SUBSET) : fparms = get_parms_for_input(parms,swsw.get_subset_run_input_idList) current_sequence = swsm.get_current_subset_sequence() current_step = swsm.get_current_subset_step() current_sequence.add_step_to_sequence_steps(current_step) if(len(fparms) > 0) : df_title = fparms[0] col_names = fparms[1] col_action = fparms[3] new_subset_step = swsm.dfc_subset_step(df_title,col_names,col_action) df = swsm.get_current_subset_df() new_subset_df = drop_add_cols(col_names,col_action,df) swsm.set_current_subset_df(new_subset_df) swsm.set_current_subset_step(new_subset_step) swsw.display_df_criteria(df_title,new_subset_df) if(DEBUG_SUBSET) : print("PROCESS_SAVE_AND_GET_SUBSET",parms,fparms) swsm.dump_current_step() swsm.dump_current_sequence() elif(optionId == swsm.PROCESS_SAVE_SUBSET) : save_subset_run(parms,0) elif(optionId == swsm.PROCESS_SUBSET_SEQUENCE) : opstat = opStatus() fparms = get_parms_for_input(parms,swsw.get_subset_sequences_input_idList) sequence = fparms[0] run_option = fparms[1] saved_sequence = swsm.get_subset_sequence(sequence) first_step = saved_sequence.get_sequence_step(0) df_title = first_step.get_input_subset_df_title() df = cfg.get_dfc_dataframe_df(df_title) if(df is None) : swsw.get_dfsubset_main_taskbar() cfg.display_data_select_df(cfg.SWDFSubsetUtility_ID) opstat.set_status(False) opstat.set_errorMsg("subset sequence starting df '" + df_title + "' is not currently loaded in dfc") display_exception(opstat) else : if(run_option == "Auto Run") : total_steps = saved_sequence.get_total_sequence_steps() swsm.set_current_subset_df(df) for i in range(total_steps) : current_step = saved_sequence.get_sequence_step(i) current_columns = current_step.get_col_names_list() columns_action = current_step.get_keep_drop_flag() criteria = current_step.get_criteria() output_df_title = current_step.get_output_subset_df_title() current_df = swsm.get_current_subset_df() if(len(current_columns) > 0) : colnames = list(current_df.columns) drop_columns = [] for i in range(len(colnames)) : if(columns_action == "Keep") : if(not (colnames[i] in current_columns)) : drop_columns.append(colnames[i]) else : if(colnames[i] in current_columns) : drop_columns.append(colnames[i]) if(len(drop_columns) > 0 ) : try : current_df.drop(drop_columns, axis=1, inplace=True) except : opstat.set_status(False) opstat.set_errorMsg("Unable to drop columns from subset dataframe") swsm.set_current_subset_df(current_df) try : current_df = swsm.get_current_subset_df() exec(criteria + swsm.starting_criteria_postamble) current_df = swsm.get_current_subset_df() except Exception as e: opstat.store_exception("Error running subset sequence '" + sequence + "'",e) swsw.display_save_subset(output_df_title,swsm.get_current_subset_df(),True) else : total_steps = saved_sequence.get_total_sequence_steps() swsm.set_current_subset_df(df) current_step = saved_sequence.get_sequence_step(0) current_df_title = current_step.get_input_subset_df_title() current_columns = current_step.get_col_names_list() columns_action = current_step.get_keep_drop_flag() swsw.display_manual_df_subset_setup(saved_sequence.get_sequence_title(),current_df_title,current_columns,columns_action,0) swsm.set_current_subset_step_id(0) swsm.set_current_subset_sequence(saved_sequence) elif(optionId == swsm.PROCESS_GET_NEXT_SUBSET) : fparms = get_parms_for_input(parms,swsw.get_manual_input_idList) collist = fparms[1] collist = collist.lstrip("[") collist = collist.rstrip("]") collist = collist.split(",") keep_drop = fparms[3] saved_sequence = swsm.get_current_subset_sequence() total_steps = saved_sequence.get_total_sequence_steps() current_step_id = swsm.get_current_subset_step_id() if(current_step_id < total_steps) : current_step = saved_sequence.get_sequence_step(swsm.get_current_subset_step_id()) else : swsm.set_current_subset_step_col_names_list(collist) swsm.set_current_subset_keep_drop_flag(keep_drop) current_step = swsm.get_current_subset_step() swsm.dump_current_step() swsm.set_current_subset_step(current_step) current_df_title = current_step.get_input_subset_df_title() current_df = swsm.get_current_subset_df() current_columns = current_step.get_col_names_list() columns_action = current_step.get_keep_drop_flag() criteria = current_step.get_criteria() output_df_title = current_step.get_output_subset_df_title() if(len(current_columns) > 0) : colnames = list(current_df.columns) drop_columns = [] for i in range(len(colnames)) : if(columns_action == "Keep") : if(not (colnames[i] in current_columns)) : drop_columns.append(colnames[i]) else : if(colnames[i] in current_columns) : drop_columns.append(colnames[i]) if(len(drop_columns) > 0 ) : try : current_df.drop(drop_columns, axis=1, inplace=True) except : opstat.set_status(False) opstat.set_errorMsg("Unable to drop columns from subset dataframe") swsw.display_next_criteria(current_df_title,current_df,criteria,output_df_title) elif(optionId == swsm.PROCESS_NEXT_CRITERIA) : opstat = opStatus() fparms = get_parms_for_input(parms,swsw.get_next_criteria_input_idList) output_df_title = fparms[0] criteria = fparms[2] current_sequence = swsm.get_subset_sequence(sequence) sequence_title = current_sequence.get_sequence_title() try : current_df = swsm.get_current_subset_df() exec(criteria + swsm.starting_criteria_postamble) current_df = swsm.get_current_subset_df() except Exception as e: opstat.store_exception("Error running subset sequence '" + sequence_title + "'",e) current_df_title = current_step.get_input_subset_df_title() current_df = swsm.get_current_subset_df() criteria = current_step.get_criteria() output_df_title = current_step.get_output_subset_df_title() swsw.display_next_criteria(current_df_title,current_df,criteria,output_df_title) display_exception(opstat) if(opstat.get_status()) : swsm.set_current_subset_df(current_df) swsm.set_current_subset_step_id(swsm.get_current_subset_step_id() + 1) if(swsm.get_current_subset_step_id() >= swsm.get_current_subset_sequence().get_total_sequence_steps()) : swsw.display_sequence_save_subset(output_df_title,swsm.get_current_subset_df()) else : current_step = swsm.get_current_subset_sequence().get_sequence_step(swsm.get_current_subset_step_id()) current_df_title = current_step.get_input_subset_df_title() current_columns = current_step.get_col_names_list() columns_action = current_step.get_keep_drop_flag() swsw.display_manual_df_subset_setup(swsm.get_current_subset_sequence().get_sequence_title(),current_df_title,current_columns,columns_action,swsm.get_current_subset_step_id()) elif(optionId == swsm.DISPLAY_NEW_STEP) : current_sequence = swsm.get_current_subset_sequence() sequence_title = current_sequence.get_sequence_title() current_step = swsm.get_current_subset_step() df_title = current_step.get_output_subset_df_title() current_df = swsm.get_current_subset_df() current_columns = [] current_action = "Keep" criteria = swsm.starting_criteria output_df_title = "" current_step = swsm.dfc_subset_step(df_title,current_columns,current_action,criteria,output_df_title) swsm.set_current_subset_step(current_step) swsw.display_manual_df_subset_setup(sequence_title,df_title,current_columns,current_action,swsm.get_current_subset_step_id()) #swsw.display_next_criteria(df_title,current_df,criteria,output_df_title) elif(optionId == swsm.PROCESS_SAVE_SAVED_SUBSET ) : save_subset_run(parms,1) elif(optionId == swsm.DISPLAY_GET_REMOTE_SUBSET) : chapterid = parms[0] new_config_df = None if(chapterid == cfg.DataInspection_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_INSPECTION_DF) elif(chapterid == cfg.DataCleansing_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_CLEANSE_DF) elif(chapterid == cfg.DataTransform_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_TRANSFORM_DF) elif(chapterid == cfg.DataExport_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_EXPORT_DF) elif(chapterid == cfg.DataImport_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_IMPORT_DF) elif(chapterid == cfg.SWGeocodeUtility_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_GEOCODE_DF) elif(chapterid == cfg.SWDFSubsetUtility_ID) : new_config_df = cfg.get_config_value(cfg.CURRENT_SUBSET_DF) cfg.set_config_value(cfg.CURRENT_SUBSET_DF,new_config_df) swsm.clear_current_subset_data() swsw.display_df_subset_setup() else : swsw.get_dfsubset_main_taskbar() cfg.drop_config_value(cfg.CURRENT_SUBSET_DF) clear_sw_utility_dfsubsetdata() cfg.display_data_select_df(cfg.SWDFSubsetUtility_ID) if(not(optionId == swsm.DISPLAY_MAIN)) : cfg.display_no_dfs(cfg.SWDFSubsetUtility_ID)
def process_export_form(formid, parms, display=True): """ * -------------------------------------------------------------------------- * function : process export function * * parms : * formid - form id * fname - export parms * display - display flag * * returns : N/A * -------------------------------------------------------- """ from dfcleanser.common.html_widgets import define_inputs, are_owner_inputs_defined if (not (are_owner_inputs_defined(cfg.DataExport_ID))): define_inputs(cfg.DataExport_ID, dew.dataexport_inputs) if (not (cfg.is_a_dfc_dataframe_loaded())): print("No Dataframe Currently Loadad") return () if (formid == dem.EXPORT_DF_FROM_CENSUS): opstat = opStatus() dfid = parms + "_df" cfg.set_config_value(cfg.CURRENT_EXPORT_DF, dfid) display_export_forms(0) elif (formid == dem.EXPORT_TO_DB_FROM_CENSUS): opstat = opStatus() dfid = parms + "_df" cfg.set_config_value(cfg.CURRENT_EXPORT_DF, dfid) dew.display_dc_export_forms(2, 4) elif ((formid == dem.CSV_EXPORT) or (formid == dem.EXCEL_EXPORT) or (formid == dem.JSON_EXPORT) or (formid == dem.HTML_EXPORT) or (formid == dem.CUSTOM_EXPORT)): opstat = opStatus() if (display): clear_output() dew.display_export_main_taskbar() save_data_export_start() clock = RunningClock() clock.start() if (formid == dem.CSV_EXPORT): fparms = dew.get_csv_export_inputs(parms) opstat = export_pandas_csv(fparms, dew.pandas_export_csv_id, dew.pandas_export_csv_labelList) parmstitle = "Pandas CSV Export Parms" parmslist = dew.pandas_export_csv_labelList[:6] elif (formid == dem.EXCEL_EXPORT): fparms = dew.get_excel_export_inputs(parms) opstat = export_pandas_excel(fparms, dew.pandas_export_excel_id, dew.pandas_export_excel_labelList) parmstitle = "Pandas Excel Export Parms" parmslist = dew.pandas_export_excel_labelList[:7] elif (formid == dem.JSON_EXPORT): fparms = dew.get_json_export_inputs(parms) opstat = export_pandas_json(fparms, dew.pandas_export_json_id, dew.pandas_export_json_labelList) parmstitle = "Pandas JSON Export Parms" parmslist = dew.pandas_export_json_labelList[:6] elif (formid == dem.HTML_EXPORT): fparms = dew.get_html_export_inputs(parms) opstat = export_pandas_html(fparms, dew.pandas_export_html_id, dew.pandas_export_html_labelList) parmstitle = "Pandas HTML Export Parms" parmslist = dew.pandas_export_html_labelList[:8] elif (formid == dem.CUSTOM_EXPORT): (dispstats, opstat) = export_custom(parms) if (dispstats): parmstitle = "Custom Export Parms" parmslist = dew.custom_export_labelList[:4] if (opstat.get_status()): if (display): if (formid == dem.CUSTOM_EXPORT): if (dispstats): ciparms = parms[0].replace("\n", "</br>") display_data_export_parms(parmstitle, parmslist, [ciparms], cfg.DataExport_ID, fparms[1], True) else: display_data_export_parms(parmstitle, parmslist, fparms, cfg.DataExport_ID, fparms[1]) else: display_exception(opstat) if (display): clock.stop() elif (formid == dem.SQLTABLE_EXPORT): export_sql_table(parms) else: print("Invalid formid " + str(formid)) return
def process_zipcode_attributes(parms): DEBUG_PROC_ZC_ATTRS = False opstat = opStatus() fparms = get_parms_for_input(parms, suzw.zipcode_atributes_input_idList) zipcode = fparms[0] cfg.set_config_value(suzw.zipcode_atributes_input_id + "Parms", fparms) suzw.display_get_zipcode_attributes(parms) print("\n") zipattrsHeader = [""] zipattrsRows = [] zipattrsWidths = [30, 70] zipattrsAligns = ["left", "left"] primary_cities = suzm.get_cities_for_zipcode( zipcode, city_type=suzm.PRIMARY_CITY_TYPE) acceptable_cities = suzm.get_cities_for_zipcode( zipcode, city_type=suzm.ACCEPTABLE_CITY_TYPE) not_acceptable_cities = suzm.get_cities_for_zipcode( zipcode, city_type=suzm.NOT_ACCEPTABLE_CITY_TYPE) zipcode_county = suzm.get_county_for_zipcode(zipcode) zipcode_state = suzm.get_state_for_zipcode(zipcode) zipcode_latitude = suzm.get_latitude_for_zipcode(zipcode) zipcode_longitude = suzm.get_longitude_for_zipcode(zipcode) zipcode_areacodes = suzm.get_areacodes_for_zipcode(zipcode) zipcode_active_status = suzm.is_zipcode_active(zipcode) zipcode_type = suzm.get_type_for_zipcode(zipcode) """ 501 603 604 8720 8732 8753 9001 9203 9204 9213 11708 """ if (DEBUG_PROC_ZC_ATTRS): print("primary_cities", primary_cities) if (not (acceptable_cities is None)): print("acceptable_cities", len(acceptable_cities), acceptable_cities) else: print("acceptable_cities", acceptable_cities) if (not (not_acceptable_cities is None)): print("not_acceptable_cities", len(not_acceptable_cities), not_acceptable_cities) else: print("not_acceptable_cities", not_acceptable_cities) print("zipcode_county", zipcode_county) print("zipcode_state", zipcode_state) print("zipcode_latitude", type(zipcode_latitude), zipcode_latitude) print("zipcode_longitude", type(zipcode_longitude), zipcode_longitude) print("zipcode_areacodes", type(zipcode_areacodes), zipcode_areacodes) print("zipcode_active_status", zipcode_active_status) print("zipcode_type", zipcode_type) if ((not (zipcode_active_status)) and (primary_cities is None)): zipattrsRows.append(["Current Status", "Zipcode is Invalid"]) else: if (zipcode_active_status): zipattrsRows.append(["Current Status", "Active"]) else: zipattrsRows.append(["Current Status", "Decommissioined"]) if (primary_cities is None): zipattrsRows.append(["Primary City", "None"]) else: if ((primary_cities == suzm.APO_ZIPCODE_TYPE) or (primary_cities == suzm.FPO_ZIPCODE_TYPE) or (primary_cities == suzm.DPO_ZIPCODE_TYPE)): zipattrsRows.append(["Primary City", "Washington DC"]) else: zipattrsRows.append(["Primary City", str(primary_cities)]) if (zipcode_county is None): zipattrsRows.append(["County", "None"]) else: zipattrsRows.append(["County", str(zipcode_county)]) if (zipcode_state is None): zipattrsRows.append(["State", "None"]) else: zipattrsRows.append(["State", str(zipcode_state)]) if ((zipcode_latitude is None) or (zipcode_longitude is None) or (numpy.isnan(zipcode_latitude)) or (numpy.isnan(zipcode_longitude))): zipattrsRows.append(["[Latitude,Longitude]", "Unknown"]) else: zipattrsRows.append([ "[Latitude,Longitude]", "[" + str(round(zipcode_latitude, 7)) + " , " + str(round(zipcode_longitude, 7)) + "]" ]) if (zipcode_type is None): zipattrsRows.append(["Zipcode Type", "None"]) else: if (zipcode_type == suzm.UNIQUE_ZIPCODE_TYPE): zipattrsRows.append(["Zipcode Type", str(suzm.UNIQUE_text)]) elif (zipcode_type == suzm.STANDARD_ZIPCODE_TYPE): zipattrsRows.append(["Zipcode Type", str(suzm.STANDARD_text)]) elif (zipcode_type == suzm.PO_BOX_ZIPCODE_TYPE): zipattrsRows.append(["Zipcode Type", str(suzm.PO_BOX_text)]) elif (zipcode_type == suzm.APO_ZIPCODE_TYPE): zipattrsRows.append(["Zipcode Type", str(suzm.APO_text)]) elif (zipcode_type == suzm.FPO_ZIPCODE_TYPE): zipattrsRows.append(["Zipcode Type", str(suzm.FPO_text)]) elif (zipcode_type == suzm.DPO_ZIPCODE_TYPE): zipattrsRows.append(["Zipcode Type", str(suzm.DPO_text)]) else: zipattrsRows.append(["Zipcode Type", "Unknown"]) if (not (zipcode_areacodes is None)): if (type(zipcode_areacodes) == list): for i in range(len(zipcode_areacodes)): zipcode_areacodes[i] = zipcode_areacodes[i].replace( "'", "") zipattrsRows.append(["Area Codes", str(zipcode_areacodes)]) if (not (acceptable_cities is None)): if (len(acceptable_cities) > 0): zipattrsRows.append( ["Acceptable Cities", str(acceptable_cities)]) if (not (not_acceptable_cities is None)): if (len(not_acceptable_cities) > 0): zipattrsRows.append( ["Not Acceptable Cities", str(not_acceptable_cities)]) zipattrs_table = None from dfcleanser.common.table_widgets import dcTable, get_row_major_table, ROW_MAJOR, SCROLL_DOWN zipattrs_table = dcTable("Zipcode " + str(zipcode) + " Properties", 'zipcodeattrsid', cfg.SWZipcodeUtility_ID, zipattrsHeader, zipattrsRows, zipattrsWidths, zipattrsAligns) zipattrs_table.set_small(True) zipattrs_table.set_checkLength(False) zipattrs_table.set_border(True) zipattrs_table.set_tabletype(ROW_MAJOR) zipattrs_table.set_rowspertable(50) zipattrsHtml = get_row_major_table(zipattrs_table, SCROLL_DOWN, False) gridclasses = ["dfc-top"] gridhtmls = [zipattrsHtml] display_generic_grid("display-geocode-coords-wrapper", gridclasses, gridhtmls)
def export_pandas_sqltable(sqltableparms, dbcondict, exportid, display=True): """ * -------------------------------------------------------------------------- * function : export pandas dataframe into sql table * * parms : * sqltableparms - export parms * dbcondict - db connector dict * exportId - export id * display - display flag * * returns : N/A * -------------------------------------------------------- """ opstat = opStatus() import dfcleanser.common.db_utils as dbu dbcon = dbu.dbConnector() from dfcleanser.common.db_utils import grab_connection_parms if (dbcondict == None): parmslist = get_stored_con_Parms( cfg.get_config_value(cfg.CURRENT_DB_ID_KEY)) dbcondict = set_dbcon_dict(cfg.get_config_value(cfg.CURRENT_DB_ID_KEY), parmslist) dbconparms = grab_connection_parms(dbcondict) else: dbconparms = grab_connection_parms(dbcondict) dbcon.set_ConnectionParms(dbconparms) dbconnector = dbcon.connect_to_db(dbu.SQLALCHEMY, opstat) if (opstat.get_status()): if (len(sqltableparms) == 0): opstat.set_status(False) opstat.set_errorMsg("No Export parameters defined") else: if (sqltableparms[0] == ""): opstat.set_status(False) opstat.set_errorMsg("No dataframe selcted to export") else: if (sqltableparms[1] == ""): opstat.set_status(False) opstat.set_errorMsg("No tabl;e selcted to export to") else: df = cfg.get_dfc_dataframe_df(sqltableparms[0]) labellist = dew.pandas_export_sqltable_labelList try: sqlkeys = [ labellist[2], labellist[3], labellist[4], labellist[5], labellist[6], labellist[7] ] sqlvals = [ sqltableparms[2], sqltableparms[3], sqltableparms[4], sqltableparms[5], sqltableparms[6], sqltableparms[7] ] sqltypes = [ STRING_PARM, STRING_PARM, BOOLEAN_PARM, STRING_PARM, INT_PARM, DICT_PARM ] sqlparms = {} sqladdlparms = {} except Exception as e: opstat.set_status(False) opstat.store_exception("Error parsing Export parms", e) if (opstat.get_status()): try: sqlparms = get_function_parms( sqlkeys, sqlvals, sqltypes) if (not (sqltableparms[8] == "")): sqladdlparms = json.loads(sqltableparms[8]) if (len(sqladdlparms) > 0): addlparmskeys = sqladdlparms.keys() for i in range(len(addlparmskeys)): sqlparms.update({ addlparmskeys[i]: sqladdlparms.get(addlparmskeys[i]) }) except Exception as e: opstat.set_status(False) opstat.store_exception( "Error parsing Export additional parms", e) if (opstat.get_status()): try: df.to_sql(sqltableparms[1], dbconnector, **sqlparms) except Exception as e: opstat.store_exception( "Unable to export to sql table", e) export_notes = "" if (opstat.get_status()): if (display): #make scriptable add_to_script([ "# Export SQL Table ", "from dfcleanser.data_export.data_export_control export export_pandas_sqltable", "export_pandas_sqltable(" + json.dumps(sqltableparms) + "," + json.dumps(dbcondict) + "," + str(exportid) + ",False)" ], opstat) export_notes = dbu.get_SQLAlchemy_connector_string(dbconparms) if (len(sqltableparms) > 0): cfg.set_config_value(exportid + "Parms", sqltableparms) cfg.set_config_value(cfg.CURRENT_EXPORTED_FILE_NAME_KEY, sqltableparms[0], True) return (export_notes, opstat)
def display_get_cities_for_county(parms): if (parms is None): fparms = cfg.get_config_value(county_cities_input_id + "Parms") if (not (fparms is None)): stateid = fparms[0] else: stateid = None else: fparms = get_parms_for_input(parms, county_cities_input_idList) stateid = fparms[0] cfg.set_config_value(county_cities_input_id + "Parms", fparms) county_cities_form = InputForm( county_cities_input_id, county_cities_input_idList, county_cities_input_labelList, county_cities_input_typeList, county_cities_input_placeholderList, county_cities_input_jsList, county_cities_input_reqList) selectDicts = [] from dfcleanser.sw_utilities.sw_utility_model import get_Dict states_dict = get_Dict("US_States_and_Territories") state_keys = list(states_dict.keys()) state_keys.sort() states_list = [] for i in range(len(state_keys)): states_list.append( str(state_keys[i]) + " : " + str(states_dict.get(state_keys[i]))) if (stateid is None): state_def = states_list[0] else: state_def = stateid state_sel = { "default": state_def, "list": states_list, "callback": "change_state_for_counties" } selectDicts.append(state_sel) state_id = state_def[:2] counties_list = suzm.get_counties_for_state(state_id) county_sel = {"default": counties_list[0], "list": counties_list} selectDicts.append(county_sel) get_select_defaults(county_cities_form, county_cities_input_id, county_cities_input_idList, county_cities_input_typeList, selectDicts) county_cities_form.set_gridwidth(720) county_cities_form.set_custombwidth(100) county_cities_form.set_fullparms(True) county_cities_input_html = "" county_cities_input_html = county_cities_form.get_html() county_cities_heading_html = "<div>Get County Cities</div>" gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"] gridhtmls = [county_cities_heading_html, county_cities_input_html] print("\n") display_generic_grid("geocode-utility-wrapper", gridclasses, gridhtmls)
def display_dict_maint(keyValue=None, loadfile=None): """ * ------------------------------------------------------------------------ * function : display the user dicts maintenance form * * parms : * keyValue - dict name * * ------------------------------------------------------------------------- """ #print("display_dict_maint",keyValue,loadfile) opstat = opStatus() if (loadfile is None): dict_maint_input_form = InputForm( maint_dict_utility_input_id, maint_dict_utility_input_idList, maint_dict_utility_input_labelList, maint_dict_utility_input_typeList, maint_dict_utility_input_placeholderList, maint_dict_utility_input_jsList, maint_dict_utility_input_reqList) else: dict_maint_input_form = InputForm( maint_dict_file_utility_input_id, maint_dict_file_utility_input_idList, maint_dict_file_utility_input_labelList, maint_dict_file_utility_input_typeList, maint_dict_file_utility_input_placeholderList, maint_dict_file_utility_input_jsList, maint_dict_file_utility_input_reqList) selectDicts = [] from dfcleanser.sw_utilities.sw_utility_model import get_dicts_names, USER_CREATED dict_names = get_dicts_names(USER_CREATED) #print("dict_names",dict_names) if (not (dict_names is None)): if (keyValue is None): def_dict = dict_names[0] else: def_dict = keyValue seldict = swum.get_Dict(def_dict, USER_CREATED) keys = list(seldict.keys()) if ((def_dict == "Country_Codes") or (def_dict == "Language_Codes")): keys.sort() seldict = swum.get_pretty_dict(seldict, keys) else: dict_names = ["No User dicts defined"] def_dict = "No User dicts defined" seldict = "User defined dict" dictssel = { "default": def_dict, "list": dict_names, "callback": "select_dict" } selectDicts.append(dictssel) from dfcleanser.common.common_utils import get_select_defaults get_select_defaults(dict_maint_input_form, maint_dict_utility_input_id, maint_dict_utility_input_idList, maint_dict_utility_input_typeList, selectDicts) dict_maint_input_form.set_gridwidth(700) #dict_maint_input_form.set_custombwidth(110) if (loadfile is None): dict_maint_input_form.set_buttonstyle({ "font-size": 13, "height": 75, "width": 90, "left-margin": 20 }) else: dict_maint_input_form.set_buttonstyle({ "font-size": 13, "height": 75, "width": 90, "left-margin": 205 }) dict_maint_input_form.set_fullparms(True) cfg.drop_config_value(maint_dict_utility_input_id + "Parms") cfg.drop_config_value(maint_dict_utility_input_id + "ParmsProtect") if (not (loadfile is None)): import json #from dfcleanser.common.common_utils import does_file_exist #print("does_file_exist",does_file_exist(loadfile)) try: with open(loadfile, 'r') as ds_file: ds = json.load(ds_file) ds_file.close() keys = list(ds.keys()) seldict = swum.get_pretty_dict(ds, keys) #print(seldict) except Exception as e: opstat.set_errorMsg("invalid user file to load " + loadfile) opstat.set_exception(e) if (opstat.get_status()): if (loadfile is None): cfg.set_config_value(maint_dict_utility_input_id + "Parms", [def_dict, "", seldict, ""]) else: cfg.set_config_value(maint_dict_utility_input_id + "Parms", [def_dict, "", seldict, loadfile]) cfg.set_config_value(maint_dict_utility_input_id + "ParmsProtect", [True, False, True, True]) help_note = "To add a user dict enter parms and values above and click on 'Add User Dict'.</br>To update the current dict change values and click on 'Update User Dict'" from dfcleanser.common.common_utils import get_help_note_html dict_maint_notes_html = get_help_note_html(help_note, 80, 75, None) dict_maint_html = "Fill in new user dict parms or update currently displayed user dict." dict_maint_html = dict_maint_input_form.get_html() dict_maint_title_html = "<div>User Dicts</div><br></br>" gridclasses = [ "dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer" ] gridhtmls = [ dict_maint_title_html, dict_maint_html, dict_maint_notes_html ] #print(dict_maint_html) #print(dict_maint_notes_html) print("\n") display_generic_grid("sw-utils-listdict-wrapper", gridclasses, gridhtmls) else: display_exception(opstat) add_error_to_log("[Get User Dict from File] " + loadfile + str(sys.exc_info()[0].__name__))
def display_inspect_rows(rowid=0): """ * -------------------------------------------------------------------------- * function : display the inspect rows option * * parms : * * returns : * N/A * -------------------------------------------------------- """ opstat = opStatus() clock = RunningClock() clock.start() try: print("\n") from dfcleanser.data_transform.data_transform_dataframe_widgets import display_current_df_index display_current_df_index( cfg.get_current_chapter_df(cfg.DataInspection_ID), cfg.get_current_chapter_dfc_df_title(cfg.DataInspection_ID), 0, True) row_stats_html = diw.display_row_stats( cfg.get_current_chapter_df(cfg.DataInspection_ID), cfg.get_config_value(cfg.CURRENT_INSPECTION_DF), False) sample_row_html = dim.display_df_rows( cfg.get_current_chapter_df(cfg.DataInspection_ID), rowid, 200) rows_openexcel_tb = diw.get_inspection_openexcel_taskbar() rows_openexcel_tb.set_gridwidth(620) rows_openexcel_tb.set_customstyle({ "font-size": 13, "height": 90, "width": 120, "left-margin": 10 }) rows_openexcel_html = rows_openexcel_tb.get_html() rows_openexcel_html = (rows_openexcel_html + "<br>") cfg.set_config_value(cfg.CURRENT_SCROLL_ROW_KEY, rowid) gridclasses = ["dfc-top", "dfc-bottom", "dfc-footer"] gridhtmls = [row_stats_html, sample_row_html, rows_openexcel_html] if (cfg.get_dfc_mode() == cfg.INLINE_MODE): display_generic_grid("df-inspection-row-data-wrapper", gridclasses, gridhtmls) else: display_generic_grid("df-inspection-row-data-pop-up-wrapper", gridclasses, gridhtmls) except Exception as e: opstat.store_exception("Error displaying row data\n ", e) display_exception(opstat) import traceback traceback.print_exc() clock.stop()
def display_dc_export_forms(exid, detid=0, notes=False): """ * -------------------------------------------------------------------------- * function : display pandas export input forms * * parms : * exid - export type * detid - detail id * notes - notes flag * * returns : N/A * -------------------------------------------------------- """ clear_output() # add the main import task bar if (exid == dem.EXPORT_TB_ONLY): display_export_main_taskbar() cfg.display_data_select_df(cfg.DataExport_ID) else: if (cfg.is_a_dfc_dataframe_loaded()): # add the pandas import task bar or pandas details form if ((exid == dem.EXPORT_PANDAS_TB_ONLY) or (exid == dem.EXPORT_PANDAS_TB_PLUS_DETAILS)): # add the pandas export details form if (exid == dem.EXPORT_PANDAS_TB_PLUS_DETAILS): if (detid == dem.SQLTABLE_EXPORT): import dfcleanser.common.db_utils as dbutils cfg.drop_config_value(pandas_export_sqltable_id + "Parms") dbid = cfg.get_config_value(cfg.CURRENT_DB_ID_KEY) if (dbid == None): cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.MySql) conparms = cfg.get_config_value( dbutils.MYSQL_DBCON_PARMS) if (conparms == None): conparms = [ "", "", "", "", dbutils.pymysql_library ] elif (dbid == dbutils.MySql): conparms = cfg.get_config_value( dbutils.MYSQL_DBCON_PARMS) if (conparms == None): conparms = [ "", "", "", "", dbutils.pymysql_library ] elif (dbid == dbutils.MS_SQL_Server): conparms = cfg.get_config_value( dbutils.MSSQL_DBCON_PARMS) if (conparms == None): conparms = [ "", "", "", "", dbutils.pyodbc_library ] elif (dbid == dbutils.SQLite): conparms = cfg.get_config_value( dbutils.SQLITE_DBCON_PARMS) if (conparms == None): conparms = ["", dbutils.sqlite3_library] elif (dbid == dbutils.Postgresql): conparms = cfg.get_config_value( dbutils.POSTGRESQL_DBCON_PARMS) if (conparms == None): conparms = [ "", "", "", "", dbutils.psycopg2_library ] elif (dbid == dbutils.Oracle): conparms = cfg.get_config_value( dbutils.ORACLE_DBCON_PARMS) if (conparms == None): conparms = [ "", "", "", dbutils.cx_oracle_library ] elif (dbid == dbutils.Custom): conparms = cfg.get_config_value( dbutils.CUSTOM_DBCON_PARMS) if (conparms == None): conparms = [""] dbutils.display_db_connector_inputs( cfg.get_config_value(cfg.CURRENT_DB_ID_KEY), conparms, dbutils.SQL_EXPORT) else: display_export_main_taskbar() pandas_export_form = get_pandas_export_input_form( detid) if (cfg.get_dfc_mode() == cfg.INLINE_MODE): pandas_export_form.set_shortForm(True) pandas_export_form.set_gridwidth(640) pandas_export_form.set_custombwidth(110) else: pandas_export_form.set_gridwidth(480) pandas_export_form.set_custombwidth(100) pandas_input_html = "" pandas_input_html = pandas_export_form.get_html() pandas_input_heading_html = "<div>" + get_pandas_export_input_title( detid) + "</div>" gridclasses = [ "dfcleanser-common-grid-header", "dfc-footer" ] gridhtmls = [ pandas_input_heading_html, pandas_input_html ] if (cfg.get_dfc_mode() == cfg.INLINE_MODE): display_generic_grid("data-import-wrapper", gridclasses, gridhtmls) else: display_generic_grid("data-import-pop-up-wrapper", gridclasses, gridhtmls) else: display_pandas_export_taskbar() elif (exid == dem.EXPORT_CUSTOM_ONLY): print("dem.EXPORT_PANDAS_TB_ONLY") # add the custom import form exportCustomDetailsForm = InputForm( custom_export_id, custom_export_idList, custom_export_labelList, custom_export_typeList, custom_export_placeholderList, custom_export_jsList, custom_export_reqList) if (notes): customNotes = [ "To create custom export code in the code cell below hit 'New Custom Export'", " (enter and test export in the code cell below)", " (leave the '# custom export' comment line in the code cell", " (call dfcleanser.common.cfg.get_dfc_dataframe_df() to get the current dataframe)", "To run the export code in the Custom Export Code box hit 'Run Custom Export' button", " (only the code in the Custom Export Code box is run and stored for scripting)", "Once import successful hit 'Save Custom Import' button to store import code for future retrieval", "To drop the custom export code and clear the Custom Export Code box hit 'Drop Custom Export' button" ] print("\n") display_inline_help(customNotes, 92) selectDicts = [] df_list = cfg.get_dfc_dataframes_select_list(cfg.DataExport_ID) selectDicts.append(df_list) flags = {"default": "False", "list": ["True", "False"]} selectDicts.append(flags) get_select_defaults(exportCustomDetailsForm, custom_export_id, custom_export_idList, custom_export_typeList, selectDicts) if (cfg.get_dfc_mode() == cfg.INLINE_MODE): exportCustomDetailsForm.set_shortForm(True) exportCustomDetailsForm.set_gridwidth(640) exportCustomDetailsForm.set_custombwidth(110) else: exportCustomDetailsForm.set_gridwidth(480) exportCustomDetailsForm.set_custombwidth(100) exportCustomDetailsForm.set_fullparms(True) from dfcleanser.common.html_widgets import new_line custom_code = "# add USER CODE to export the df" + new_line + new_line custom_code = ( custom_code + "from dfcleanser.common.cfg import get_dfc_dataframe" + new_line) custom_code = (custom_code + "df = get_dfc_dataframe_df(dataframe_title)" + new_line + new_line) custom_code = (custom_code + "# USER CODE" + new_line) cfg.set_config_value(custom_export_id + "Parms", ["", custom_code, ""]) custom_export_html = "" custom_export_html = exportCustomDetailsForm.get_html() custom_export_heading_html = "<div>Custom Export</div><br>" gridclasses = ["dfcleanser-common-grid-header", "dfc-footer"] gridhtmls = [custom_export_heading_html, custom_export_html] if (cfg.get_dfc_mode() == cfg.INLINE_MODE): display_generic_grid("data-import-wrapper", gridclasses, gridhtmls) else: display_generic_grid("data-import-pop-up-wrapper", gridclasses, gridhtmls) else: display_export_main_taskbar() cfg.display_data_select_df(cfg.DataExport_ID) cfg.display_no_dfs(cfg.DataExport_ID)
def display_funcs(): """ * ------------------------------------------------------------------------ * function : display the dfc funcs start form * * parms : * * ------------------------------------------------------------------------- """ list_funcs_input_form = InputForm( build_funcs_utility_input_id, build_funcs_utility_input_idList, build_funcs_utility_input_labelList, build_funcs_utility_input_typeList, build_funcs_utility_input_placeholderList, build_funcs_utility_input_jsList, build_funcs_utility_input_reqList) selectDicts = [] from dfcleanser.sw_utilities.sw_utility_genfunc_model import get_genfunc_list gen_funcs = get_genfunc_list() default_func = gen_funcs[0] funcssel = { "default": default_func, "list": gen_funcs, "callback": "select_gen_func" } selectDicts.append(funcssel) from dfcleanser.common.common_utils import get_select_defaults get_select_defaults(list_funcs_input_form, build_funcs_utility_input_id, build_funcs_utility_input_idList, build_funcs_utility_input_typeList, selectDicts) list_funcs_input_form.set_gridwidth(700) list_funcs_input_form.set_custombwidth(120) list_funcs_input_form.set_fullparms(True) from dfcleanser.sw_utilities.sw_utility_genfunc_model import reservedfunctionsmodule from dfcleanser.sw_utilities.sw_utility_genfunc_functions import get_generic_function_code func_code = get_generic_function_code(default_func) cfg.set_config_value(build_funcs_utility_input_id + "Parms", [default_func, reservedfunctionsmodule, func_code]) cfg.set_config_value(build_funcs_utility_input_id + "ParmsProtect", [False, True, True]) help_note = "Generic functions help note'" from dfcleanser.common.common_utils import get_help_note_html list_funcs_notes_html = get_help_note_html(help_note, 100, None, None) list_funcs_html = "" list_funcs_html = list_funcs_input_form.get_html() list_funcs_title_html = "<div>Generic dfc Functions</div><br></br>" gridclasses = ["dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer"] gridhtmls = [list_funcs_title_html, list_funcs_html, list_funcs_notes_html] print("\n") display_generic_grid("sw-utils-listdict-wrapper", gridclasses, gridhtmls)
def get_df_list_html(title): """ * -------------------------------------------------------------------------- * function : get the html for list of dfc dataframes * * parms : * N/A * * returns : N/A * -------------------------------------------------------- """ fparms = [] if (title is None): if (cfg.is_a_dfc_dataframe_loaded()): df_titles = cfg.get_dfc_dataframes_titles_list() fparms = [ df_titles[0], str(len(cfg.get_dfc_dataframe_df(df_titles[0]))), str(len(cfg.get_dfc_dataframe_df(df_titles[0]).columns)), cfg.get_dfc_dataframe_notes(df_titles[0]) ] else: fparms = ["", "", "", ""] else: if (cfg.is_a_dfc_dataframe_loaded()): dfc_df = cfg.get_dfc_dataframe(title) if (dfc_df is None): fparms = ["", "", "", ""] else: fparms = [ title, str(len(cfg.get_dfc_dataframe_df(title))), str(len(cfg.get_dfc_dataframe_df(title).columns)), cfg.get_dfc_dataframe_notes(title) ] else: fparms = ["", "", "", ""] parmsProtect = [False, True, True, False] cfg.set_config_value(dfmgr_input_id + "Parms", fparms) cfg.set_config_value(dfmgr_input_id + "ParmsProtect", parmsProtect) dfmanager_input_form = InputForm(dfmgr_input_id, dfmgr_input_idList, dfmgr_input_labelList, dfmgr_input_typeList, dfmgr_input_placeholderList, dfmgr_input_jsList, dfmgr_input_reqList) selectDicts = [] df_titles = cfg.get_dfc_dataframes_titles_list() #if(df_titles is None) : if (not (cfg.is_a_dfc_dataframe_loaded())): dfs = {"default": " ", "list": [" "]} else: dfs = { "default": str(fparms[0]), "list": df_titles, "callback": "select_new_df" } selectDicts.append(dfs) get_select_defaults(dfmanager_input_form, dfmgr_input_id, dfmgr_input_idList, dfmgr_input_typeList, selectDicts) dfmanager_input_form.set_shortForm(True) dfmanager_input_form.set_gridwidth(480) dfmanager_input_form.set_custombwidth(90) dfmanager_input_form.set_fullparms(True) dfmgr_input_html = dfmanager_input_form.get_html() return (dfmgr_input_html)
def display_list_maint(keyValue=None, loadfile=None): """ * ------------------------------------------------------------------------ * function : display the user lists maintenance form * * parms : * keyValue - list name * * ------------------------------------------------------------------------- """ opstat = opStatus() if (loadfile is None): list_maint_input_form = InputForm( maint_list_utility_input_id, maint_list_utility_input_idList, maint_list_utility_input_labelList, maint_list_utility_input_typeList, maint_list_utility_input_placeholderList, maint_list_utility_input_jsList, maint_list_utility_input_reqList) else: list_maint_input_form = InputForm( maint_list_file_utility_input_id, maint_list_file_utility_input_idList, maint_list_file_utility_input_labelList, maint_list_file_utility_input_typeList, maint_list_file_utility_input_placeholderList, maint_list_file_utility_input_jsList, maint_list_file_utility_input_reqList) selectDicts = [] from dfcleanser.sw_utilities.sw_utility_model import get_lists_names, USER_CREATED list_names = get_lists_names(USER_CREATED) #print("list_names",list_names) if (not (list_names is None)): if (keyValue is None): def_list = list_names[0] else: def_list = keyValue sellist = swum.get_List(def_list, USER_CREATED) dsstr = "[" for i in range(len(sellist)): dsstr = dsstr + str(sellist[i]) if (i == (len(sellist) - 1)): dsstr = dsstr + "]" else: dsstr = dsstr + "," else: list_names = ["No User lists defined"] def_list = "No User lists defined" sellist = "User defined list" listssel = { "default": def_list, "list": list_names, "callback": "select_list" } selectDicts.append(listssel) from dfcleanser.common.common_utils import get_select_defaults get_select_defaults(list_maint_input_form, maint_list_utility_input_id, maint_list_utility_input_idList, maint_list_utility_input_typeList, selectDicts) list_maint_input_form.set_gridwidth(700) if (loadfile is None): list_maint_input_form.set_buttonstyle({ "font-size": 13, "height": 75, "width": 90, "left-margin": 20 }) else: list_maint_input_form.set_buttonstyle({ "font-size": 13, "height": 75, "width": 90, "left-margin": 205 }) list_maint_input_form.set_fullparms(True) cfg.drop_config_value(maint_list_utility_input_id + "Parms") cfg.drop_config_value(maint_list_utility_input_id + "ParmsProtect") if (not (loadfile is None)): import json from dfcleanser.common.common_utils import does_file_exist if (does_file_exist(loadfile)): try: with open(loadfile, 'r') as ds_file: ds = json.load(ds_file) ds_file.close() dsstr = "[" for i in range(len(ds)): dsstr = dsstr + str(ds[i]) if (i == (len(ds) - 1)): dsstr = dsstr + "]" else: dsstr = dsstr + "," except Exception as e: opstat.set_status(False) opstat.set_errorMsg("Error processing user file to load" + loadfile) opstat.set_exception(e) else: opstat.set_status(False) opstat.set_errorMsg("invalid user file to load" + loadfile) if (opstat.get_status()): if (loadfile is None): cfg.set_config_value(maint_list_utility_input_id + "Parms", [def_list, "", dsstr, ""]) else: cfg.set_config_value(maint_list_utility_input_id + "Parms", [def_list, "", dsstr, loadfile]) cfg.set_config_value(maint_list_utility_input_id + "ParmsProtect", [True, False, True, True]) help_note = "To add a user list enter parms and values above and click on 'Add User List'.</br>To update the current list change values and click on 'Update User List'" from dfcleanser.common.common_utils import get_help_note_html list_maint_notes_html = get_help_note_html(help_note, 80, 75, None) list_maint_html = "" list_maint_html = list_maint_input_form.get_html() list_maint_title_html = "<div>User Lists</div><br></br>" gridclasses = [ "dfcleanser-common-grid-header", "dfc-bottom", "dfc-footer" ] gridhtmls = [ list_maint_title_html, list_maint_html, list_maint_notes_html ] #print(list_maint_html) #print(list_maint_notes_html) print("\n") display_generic_grid("sw-utils-listdict-wrapper", gridclasses, gridhtmls) else: display_exception(opstat) add_error_to_log("[Get User Dict from File] " + loadfile + str(sys.exc_info()[0].__name__))
def display_export_dc_sql_details_forms(dblibid): """ * -------------------------------------------------------------------------- * function : display export sql form * * parms : * dblibid - db loib id * * returns : N/A * -------------------------------------------------------- """ import dfcleanser.common.db_utils as dbutils if (dblibid == None): cfg.drop_config_value(cfg.CURRENT_DB_ID_KEY) if ((dblibid == dbutils.pymysql_library) or (dblibid == dbutils.mysql_connector_library)): cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.MySql) elif ((dblibid == dbutils.pyodbc_library) or (dblibid == dbutils.pymssql_library)): cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.MS_SQL_Server) elif ((dblibid == dbutils.sqlite_library) or (dblibid == dbutils.sqlite3_library)): cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.SQLite) elif (dblibid == dbutils.psycopg2_library): cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.Postgresql) elif (dblibid == dbutils.cx_oracle_library): cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.Oracle) elif (dblibid == "custom"): cfg.set_config_value(cfg.CURRENT_DB_ID_KEY, dbutils.Custom) dbid = cfg.get_config_value(cfg.CURRENT_DB_ID_KEY) if (dbid == None): inparms = cfg.get_config_value(cfg.MYSQL_IMPORT_PARMS_KEY) if (inparms == None): inparms = ["", "", "", "", dblibid] else: inparms[4] = dblibid if (dbid == dbutils.MySql): inparms = cfg.get_config_value(cfg.MYSQL_IMPORT_PARMS_KEY) if (inparms == None): inparms = ["", "", "", "", dblibid] else: inparms[4] = dblibid elif (dbid == dbutils.MS_SQL_Server): inparms = cfg.get_config_value(cfg.MSSQL_IMPORT_PARMS_KEY) if (inparms == None): inparms = ["", "", "", "", dblibid] else: inparms[4] = dblibid elif (dbid == dbutils.Postgresql): inparms = cfg.get_config_value(cfg.POSTGRESQL_IMPORT_PARMS_KEY) if (inparms == None): inparms = ["", "", "", "", dblibid] else: inparms[4] = dblibid elif (dbid == dbutils.SQLite): inparms = cfg.get_config_value(cfg.SQLITE_IMPORT_PARMS_KEY) if (inparms == None): inparms = ["", dblibid] else: inparms[1] = dblibid elif (dbid == dbutils.Oracle): inparms = cfg.get_config_value(cfg.ORACLE_IMPORT_PARMS_KEY) if (inparms == None): inparms = ["", "", "", dblibid] else: inparms[1] = dblibid elif (dbid == dbutils.Custom): inparms = cfg.get_config_value(cfg.CUSTOM_IMPORT_PARMS_KEY) from dfcleanser.common.db_utils import display_db_connector_inputs, SQL_EXPORT display_db_connector_inputs(cfg.get_config_value(cfg.CURRENT_DB_ID_KEY), inparms, SQL_EXPORT)