def test_linear_regression_model(self): train_file = FileUtils.path('', 'iris2.csv') test_file = FileUtils.path('', 'test_iris.csv') iris = pd.read_csv(train_file) test_iris = pd.read_csv(test_file) instanceOfLR, summary = linearClassifier(iris, test_iris, 4, 0.05, 200) instanceOfLR1, summary1 = linearClassifier(iris, test_iris, 4, 0.01, 500) instanceOfLR2, summary2 = linearClassifier(test_iris, iris, 4, 0.1, 100) print(instanceOfLR) print(summary) x = numpy.array([5.2, 3.5, 1.5, 0.2]) print(instanceOfLR.predict(x)) print(instanceOfLR1.numberOfInstance()) print(instanceOfLR1) print(summary1) x = numpy.array([5.2, 3.5, 1.4, 0.2]) print(instanceOfLR1.predict(x)) print(instanceOfLR1.numberOfInstance()) print(instanceOfLR2) print(summary2) x = numpy.array([5.2, 3.5, 1.6, 0.2]) print(instanceOfLR2.predict(x)) print(instanceOfLR2.numberOfInstance())
def display_data(value): """Displaying the head for the selected file.""" db_value = db.get("file") if value is None and db_value is None: return "" elif value is None and not db_value is None: value = db_value elif not value == db_value: db.reset() format = FileUtils.file_format(value) if format == 'csv' or format == 'txt': path = FileUtils.path('raw', value) head = DataUtils.read_text_head(path) table_col = [html.Col(style = {'width':"10%"}), html.Col(style = {'width':"90%"})] table_header = [html.Thead(html.Tr([html.Th("Row No"), html.Th("Data")]))] rows = [] for i in range(len(head)): row = html.Tr([html.Td(i+1), html.Td(head[i])]) rows.append(row) table_body = [html.Tbody(rows)] table = dbc.Table(table_col+ table_header + table_body, bordered=True, style = common.table_style) div = [common.msg("Selected File: " + value), common.msg("Selected Format: " + format), table, html.Br(), csv_properties_div] elif format == 'jpeg' or format == 'jpg' or format == 'gif': div = [common.msg("Selected File: " + value), common.msg("Selected Format: " + format)] else: div = "Format Not Supported!!" db.put("file", value) db.put("format", format) return div
def upload_data(contents, filename): """Upload Files and Regenerate the file list.""" if contents: for i in range(len(filename)): FileUtils.upload(filename[i], contents[i]) files = FileUtils.files('raw') if len(files) == 0: options = [{'label': 'No files uploaded yet!', 'value': 'None'}] return options else: options = [{'label': file, 'value': file} for file in files] return options
def read(dir: str, filename: str): format = FileUtils.file_format(filename) path = FileUtils.path(dir, filename) op = None if format == 'csv' or format == 'txt': with open(path) as myfile: head = [next(myfile).strip() for x in range(N)] op = head elif format == 'jpeg' or format == 'jpg' or format == 'gif': "" else: op = "Format Not Supported!!" return op
def test_image_recog_eight(self): img_path = FileUtils.path('images', 'eight.png') net = DigitNeuralNet1HiddenLayer(784, 100, 10) net.load() q = net.predict(img_path) self.assertEqual(q, 3)
def load(self): dir = os.path.join('nets', 'digit_1_hidden_layer') params_file_path = FileUtils.path(dir, 'params.json') con_mat_file_path = FileUtils.path(dir, 'con_mat.json') wih_file_path = FileUtils.path(dir, 'wih.csv') who_file_path = FileUtils.path(dir, 'who.csv') out_param_file = open(params_file_path) self.params = json.load(out_param_file) out_param_file.close() out_con_file = open(con_mat_file_path) self.con_mat = json.load(out_con_file) out_con_file.close() self.wih = loadtxt(wih_file_path, delimiter=',') self.who = loadtxt(who_file_path, delimiter=',') pass
def apply_file_properties(n): file = db.get("file") format = db.get("format") sep = db.get("file_separator") header = db.get("file_header") div = None if format is None: div = None elif (format == 'csv' or format == 'txt') and header is None: div = common.error_msg('Please Select Header!!') elif format == 'csv' or format == 'txt': if sep is None: sep = ',' db.put("file_separator", sep) path = FileUtils.path('raw', file) df = DataUtils.read_csv(path, sep, header) db.put("data", df) msg = "Following Properties Applied. Separator=" + sep + " Header=" + str( header) table = dbc.Table.from_dataframe(df.head(10), striped=True, bordered=True, hover=True, style=common.table_style) div = [common.msg(msg), table] return div
def read_text_head(path: str): format = FileUtils.file_format(path) op = None if format == 'csv' or format == 'txt': with open(path) as myfile: head = [next(myfile).strip() for x in range(N)] op = head return op
def dtn_display_selected_file_scatter_plot(value): value = "banknote" db.put("dtn.file", value) file = value path = FileUtils.path('clean', file) df = DataUtils.read_csv(path) save_path = FileUtils.path('extra', 'banknote.csv') df.to_csv(save_path, index=False, header = False) db.put("dtn.data", df) db.put('dtn.model_class', 'class') db.put('dtn.model_variables', ['variance','skewness','curtosis','entropy']) call_path = FileUtils.path('nets', 'dt_banknote_call1.csv') cdf = DataUtils.read_csv(call_path) trace_1 = go.Scatter(x = cdf['max_depth'], y = cdf['avg_train_score'], name = 'Average Train Score') trace_2 = go.Scatter(x = cdf['max_depth'], y = cdf['avg_test_score'], name = 'Average Test Score') title = go.Layout(title = 'Depth of Tree Vs Performance Plot', hovermode = 'closest', xaxis={'title': 'Depth of Tree'}, yaxis={'title': 'Performance'}) fig = go.Figure(data = [trace_1, trace_2], layout = title) div = html.Div([ common.msg("Selected cleaned file: "+ file), dbc.Table.from_dataframe(df.head(10).round(5).astype(str), striped=True, bordered=True, hover=True, style = common.table_style), html.Br(), html.H2('Using Default parameters for both max_depth and min_size.'), html.H2('Max Depth = 2 to 15'), html.H2('Min Size = 10'), dbc.Table.from_dataframe(cdf.round(4), striped=True, bordered=True, hover=True, style = common.table_style), html.Br(), dcc.Graph(id='dtn-plot', figure=fig), html.Br(), get_dtn_model_properties_div(df), dcc.Loading(id="dtn-model-training", children=[html.Div([], id = "dtn-trained-model", style = {'margin': '10px'})], type="default"), ]) return div
def save(self): dir = os.path.join('nets', 'digit_2_hidden_layer') FileUtils.mkdir(dir) params_file_path = FileUtils.path(dir, 'params.json') con_mat_file_path = FileUtils.path(dir, 'con_mat.json') wih_file_path = FileUtils.path(dir, 'wih.csv') whh_file_path = FileUtils.path(dir, 'whh.csv') who_file_path = FileUtils.path(dir, 'who.csv') out_param_file = open(params_file_path, "w") json.dump(self.params, out_param_file) out_param_file.close() out_con_file = open(con_mat_file_path, "w") json.dump(self.con_mat, out_con_file) out_con_file.close() savetxt(wih_file_path, self.wih, delimiter=',') savetxt(whh_file_path, self.whh2, delimiter=',') savetxt(who_file_path, self.who, delimiter=',') pass
def dtn_model_train(n_clicks): c = db.get('dtn.model_class') var = db.get('dtn.model_variables') max_depth = db.get('dtn.max_depth') min_size = db.get('dtn.min_size') folds = 5 if c is None or var is None or max_depth is None or min_size is None: div = "" elif (not c is None) and (not var is None) and (not max_depth is None) and (not min_size is None): try: path = FileUtils.path('extra', 'banknote.csv') tree, avg_score, avg_f1_score = train(path, max_depth, min_size, folds) summary = {} summary['Max Depth'] = max_depth summary['Min Size'] = min_size summary['Folds'] = folds summary['Average Score'] = round(avg_score, 4) summary['Average F1 Score'] = round(avg_f1_score, 4) summary_df = pd.DataFrame(summary.items(), columns=['Parameters', 'Value']) db.put('dtn.model_summary', summary) db.put('dtn.model_instance', tree) except Exception as e: traceback.print_exc() return common.error_msg("Exception during training model: " + str(e)) div = html.Div([ html.H2('Model Parameters & Summary:'), dbc.Table.from_dataframe(summary_df, striped=True, bordered=True, hover=True, style = common.table_style), html.Br(), html.H2('Tree'), html.H2(str(tree)), ]) else: div = common.error_msg('Select Proper Model Parameters!!') return div
def dt_display_selected_file_scatter_plot(value): db_value = db.get("dt.file") if value is None and db_value is None: return common.msg("Please select a cleaned file to proceed!!") elif value is None and not db_value is None: value = db_value db.put("dt.file", value) file = value path = FileUtils.path('clean', file) df = DataUtils.read_csv(path) db.put("dt.data", df) div = html.Div([ common.msg("Selected cleaned file: "+ file), dbc.Table.from_dataframe(df.head(10).astype(str), striped=True, bordered=True, hover=True, style = common.table_style), #html.Div([html.H3("Data Statistics")], style={'width': '100%', 'display': 'flex', 'align-items': 'center', 'justify-content': 'center'}), #dbc.Table.from_dataframe(stats, striped=True, bordered=True, hover=True, style = common.table_style), html.Br(), get_dt_model_properties_div(df), html.Div([], id = "dt-trained-model", style = {'margin': '10px'}), ]) return div
def upload_data(contents, filename): """Upload Files and Regenerate the file list.""" if contents: for i in range(len(filename)): FileUtils.upload(filename[i], contents[i]) return common.get_options('raw')
def nlcl_display_selected_file_scatter_plot(value): db_value = db.get("nlcl.file") if value is None and db_value is None: return common.msg("Please select a cleaned file to proceed!!") elif value is None and not db_value is None: value = db_value db.put("nlcl.file", value) file = value path = FileUtils.path('clean', file) df = DataUtils.read_csv(path) db.put("nlcl.data", df) stats = df.describe(include = 'all').head(6).round(5) stats.insert(loc=0, column='Statistics', value=['Count','unique','top','freq','Mean','Standard Deviation']) stats = stats.drop(stats.index[[1,2,3]]) div = html.Div([ common.msg("Selected cleaned file: "+ file), dbc.Table.from_dataframe(df.head(10), striped=True, bordered=True, hover=True, style = common.table_style), html.Div([html.H3("Data Statistics")], style={'width': '100%', 'display': 'flex', 'align-items': 'center', 'justify-content': 'center'}), dbc.Table.from_dataframe(stats, striped=True, bordered=True, hover=True, style = common.table_style), html.Br(), html.Div([html.H2("Scatter Plot")], style={'width': '100%', 'display': 'flex', 'align-items': 'center', 'justify-content': 'center'}), dbc.Row([ dbc.Col([ dbc.Label("Select Class"), dcc.Dropdown( id = 'nlcl-class', options=[{'label':col, 'value':col} for col in [*df]], value=None, multi=False ), html.Br(), dbc.Label("Select X Axis"), dcc.Dropdown( id = 'nlcl-x-axis', options=[{'label':col, 'value':col} for col in [*df]], value=None, multi=False ), html.Br(), dbc.Label("Select Y Axis"), dcc.Dropdown( id = 'nlcl-y-axis', options=[{'label':col, 'value':col} for col in [*df]], value=None, multi=False ), html.Br(), dbc.Button("Plot", color="primary", id = 'nlcl-scatter-plot-button'), html.Div([], id = "nlcl-class-do-nothing"), html.Div([], id = "nlcl-x-axis-do-nothing"), html.Div([], id = "nlcl-y-axis-do-nothing") ], md=2, style = {'margin': '10px', 'font-size': '16px'}), dbc.Col([], md=9, id="nlcl-scatter-plot") ]), html.Br(), get_nlcl_model_properties_div(df), html.Div([], id = "nlcl-trained-model", style = {'margin': '10px'}), ]) return div
################################# # This has to be added to find the custom packages import sys import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + '/../../') ################################# import zipfile from ml.neural_net.digit_recog_1_layer import DigitNeuralNet1HiddenLayer from ml.framework.file_utils import FileUtils train_data_zip_path = FileUtils.path('', 'mnist_train.csv.zip') directory_to_extract_to = FileUtils.path('extra', '') with zipfile.ZipFile(train_data_zip_path, 'r') as zip_ref: zip_ref.extractall(directory_to_extract_to) test_data_zip_path = FileUtils.path('', 'mnist_test.csv.zip') with zipfile.ZipFile(test_data_zip_path, 'r') as zip_ref: zip_ref.extractall(directory_to_extract_to) #Neural Net Training input_nodes = 784 hidden_nodes = 100 output_nodes = 10 learning_rate = 0.1 epoch = 5 train_data_path = FileUtils.path('extra', 'mnist_train.csv') test_data_path = FileUtils.path('extra', 'mnist_test.csv')
"""App Main python3 app_main """ import os from ml.framework.file_utils import FileUtils FileUtils.mkdir('raw') FileUtils.mkdir('clean') FileUtils.mkdir('knn') from ml.ux import index if __name__ == '__main__': ""
"""App Main python3 app_main """ import os from ml.framework.file_utils import FileUtils FileUtils.mkdir('raw') from ml.ux import index if __name__ == '__main__': ""
'width': '50%', 'height': '50px', 'lineHeight': '50px', 'borderWidth': '1px', 'borderStyle': 'dashed', 'borderRadius': '5px', 'textAlign': 'center', 'margin': '10px' }, multiple=True), html.Br(), html.Div([ html.H2("Select a file from all the uploaded files:"), dcc.Dropdown( id = 'selected-file', options=[{'label':file, 'value':file} for file in FileUtils.files('raw')], value=None, multi=False ), html.Br(), ], style = {'margin': '10px', 'width': '50%'}), html.Div([], id = "display-file"), html.Div([], id = "file-properties"), html.Div([], id = "file-separator-do-nothing"), html.Div([], id = "file-header-do-nothing") ]) @app.callback( Output("selected-file", "options"), [Input('upload-data', 'contents'),
'lineHeight': '50px', 'borderWidth': '1px', 'borderStyle': 'dashed', 'borderRadius': '5px', 'textAlign': 'center', 'margin': '10px' }, multiple=True), html.Br(), html.Div([ html.H2("Select a file from all the uploaded files:"), dcc.Dropdown(id='selected-file', options=[{ 'label': file, 'value': file } for file in FileUtils.files('raw')], value=None, multi=False), html.Br(), ], style={ 'margin': '10px', 'width': '50%' }), html.Div([], id="display-file"), html.Div([], id="file-properties"), html.Div([], id="file-separator-do-nothing"), html.Div([], id="file-header-do-nothing") ])
def get_options(dir: str): files = FileUtils.files(dir) if len(files) == 0: return [{'label': 'No files yet!', 'value': 'None'}] else: return [{'label': file, 'value': file} for file in files]