def test_linear_regression_model(self):
train_file = FileUtils.path('', 'iris2.csv')
test_file = FileUtils.path('', 'test_iris.csv')
iris = pd.read_csv(train_file)
test_iris = pd.read_csv(test_file)
instanceOfLR, summary = linearClassifier(iris, test_iris, 4, 0.05, 200)
instanceOfLR1, summary1 = linearClassifier(iris, test_iris, 4, 0.01,
500)
instanceOfLR2, summary2 = linearClassifier(test_iris, iris, 4, 0.1,
100)
print(instanceOfLR)
print(summary)
x = numpy.array([5.2, 3.5, 1.5, 0.2])
print(instanceOfLR.predict(x))
print(instanceOfLR1.numberOfInstance())
print(instanceOfLR1)
print(summary1)
x = numpy.array([5.2, 3.5, 1.4, 0.2])
print(instanceOfLR1.predict(x))
print(instanceOfLR1.numberOfInstance())
print(instanceOfLR2)
print(summary2)
x = numpy.array([5.2, 3.5, 1.6, 0.2])
print(instanceOfLR2.predict(x))
print(instanceOfLR2.numberOfInstance())
def display_data(value):
"""Displaying the head for the selected file."""
db_value = db.get("file")
if value is None and db_value is None:
return ""
elif value is None and not db_value is None:
value = db_value
elif not value == db_value:
db.reset()
format = FileUtils.file_format(value)
if format == 'csv' or format == 'txt':
path = FileUtils.path('raw', value)
head = DataUtils.read_text_head(path)
table_col = [html.Col(style = {'width':"10%"}), html.Col(style = {'width':"90%"})]
table_header = [html.Thead(html.Tr([html.Th("Row No"), html.Th("Data")]))]
rows = []
for i in range(len(head)):
row = html.Tr([html.Td(i+1), html.Td(head[i])])
rows.append(row)
table_body = [html.Tbody(rows)]
table = dbc.Table(table_col+ table_header + table_body, bordered=True, style = common.table_style)
div = [common.msg("Selected File: " + value),
common.msg("Selected Format: " + format),
table,
html.Br(),
csv_properties_div]
elif format == 'jpeg' or format == 'jpg' or format == 'gif':
div = [common.msg("Selected File: " + value),
common.msg("Selected Format: " + format)]
else:
div = "Format Not Supported!!"
db.put("file", value)
db.put("format", format)
return div
def upload_data(contents, filename):
"""Upload Files and Regenerate the file list."""
if contents:
for i in range(len(filename)):
FileUtils.upload(filename[i], contents[i])
files = FileUtils.files('raw')
if len(files) == 0:
options = [{'label': 'No files uploaded yet!', 'value': 'None'}]
return options
else:
options = [{'label': file, 'value': file} for file in files]
return options
def read(dir: str, filename: str):
format = FileUtils.file_format(filename)
path = FileUtils.path(dir, filename)
op = None
if format == 'csv' or format == 'txt':
with open(path) as myfile:
head = [next(myfile).strip() for x in range(N)]
op = head
elif format == 'jpeg' or format == 'jpg' or format == 'gif':
""
else:
op = "Format Not Supported!!"
return op
def test_image_recog_eight(self):
img_path = FileUtils.path('images', 'eight.png')
net = DigitNeuralNet1HiddenLayer(784, 100, 10)
net.load()
q = net.predict(img_path)
self.assertEqual(q, 3)
def load(self):
dir = os.path.join('nets', 'digit_1_hidden_layer')
params_file_path = FileUtils.path(dir, 'params.json')
con_mat_file_path = FileUtils.path(dir, 'con_mat.json')
wih_file_path = FileUtils.path(dir, 'wih.csv')
who_file_path = FileUtils.path(dir, 'who.csv')
out_param_file = open(params_file_path)
self.params = json.load(out_param_file)
out_param_file.close()
out_con_file = open(con_mat_file_path)
self.con_mat = json.load(out_con_file)
out_con_file.close()
self.wih = loadtxt(wih_file_path, delimiter=',')
self.who = loadtxt(who_file_path, delimiter=',')
pass
def apply_file_properties(n):
file = db.get("file")
format = db.get("format")
sep = db.get("file_separator")
header = db.get("file_header")
div = None
if format is None:
div = None
elif (format == 'csv' or format == 'txt') and header is None:
div = common.error_msg('Please Select Header!!')
elif format == 'csv' or format == 'txt':
if sep is None:
sep = ','
db.put("file_separator", sep)
path = FileUtils.path('raw', file)
df = DataUtils.read_csv(path, sep, header)
db.put("data", df)
msg = "Following Properties Applied. Separator=" + sep + " Header=" + str(
header)
table = dbc.Table.from_dataframe(df.head(10),
striped=True,
bordered=True,
hover=True,
style=common.table_style)
div = [common.msg(msg), table]
return div
def read_text_head(path: str):
format = FileUtils.file_format(path)
op = None
if format == 'csv' or format == 'txt':
with open(path) as myfile:
head = [next(myfile).strip() for x in range(N)]
op = head
return op
def dtn_display_selected_file_scatter_plot(value):
value = "banknote"
db.put("dtn.file", value)
file = value
path = FileUtils.path('clean', file)
df = DataUtils.read_csv(path)
save_path = FileUtils.path('extra', 'banknote.csv')
df.to_csv(save_path, index=False, header = False)
db.put("dtn.data", df)
db.put('dtn.model_class', 'class')
db.put('dtn.model_variables', ['variance','skewness','curtosis','entropy'])
call_path = FileUtils.path('nets', 'dt_banknote_call1.csv')
cdf = DataUtils.read_csv(call_path)
trace_1 = go.Scatter(x = cdf['max_depth'], y = cdf['avg_train_score'], name = 'Average Train Score')
trace_2 = go.Scatter(x = cdf['max_depth'], y = cdf['avg_test_score'], name = 'Average Test Score')
title = go.Layout(title = 'Depth of Tree Vs Performance Plot', hovermode = 'closest', xaxis={'title': 'Depth of Tree'}, yaxis={'title': 'Performance'})
fig = go.Figure(data = [trace_1, trace_2], layout = title)
div = html.Div([
common.msg("Selected cleaned file: "+ file),
dbc.Table.from_dataframe(df.head(10).round(5).astype(str), striped=True, bordered=True, hover=True, style = common.table_style),
html.Br(),
html.H2('Using Default parameters for both max_depth and min_size.'),
html.H2('Max Depth = 2 to 15'),
html.H2('Min Size = 10'),
dbc.Table.from_dataframe(cdf.round(4), striped=True, bordered=True, hover=True, style = common.table_style),
html.Br(),
dcc.Graph(id='dtn-plot', figure=fig),
html.Br(),
get_dtn_model_properties_div(df),
dcc.Loading(id="dtn-model-training",
children=[html.Div([], id = "dtn-trained-model", style = {'margin': '10px'})],
type="default"),
])
return div
def save(self):
dir = os.path.join('nets', 'digit_2_hidden_layer')
FileUtils.mkdir(dir)
params_file_path = FileUtils.path(dir, 'params.json')
con_mat_file_path = FileUtils.path(dir, 'con_mat.json')
wih_file_path = FileUtils.path(dir, 'wih.csv')
whh_file_path = FileUtils.path(dir, 'whh.csv')
who_file_path = FileUtils.path(dir, 'who.csv')
out_param_file = open(params_file_path, "w")
json.dump(self.params, out_param_file)
out_param_file.close()
out_con_file = open(con_mat_file_path, "w")
json.dump(self.con_mat, out_con_file)
out_con_file.close()
savetxt(wih_file_path, self.wih, delimiter=',')
savetxt(whh_file_path, self.whh2, delimiter=',')
savetxt(who_file_path, self.who, delimiter=',')
pass
def dtn_model_train(n_clicks):
c = db.get('dtn.model_class')
var = db.get('dtn.model_variables')
max_depth = db.get('dtn.max_depth')
min_size = db.get('dtn.min_size')
folds = 5
if c is None or var is None or max_depth is None or min_size is None:
div = ""
elif (not c is None) and (not var is None) and (not max_depth is None) and (not min_size is None):
try:
path = FileUtils.path('extra', 'banknote.csv')
tree, avg_score, avg_f1_score = train(path, max_depth, min_size, folds)
summary = {}
summary['Max Depth'] = max_depth
summary['Min Size'] = min_size
summary['Folds'] = folds
summary['Average Score'] = round(avg_score, 4)
summary['Average F1 Score'] = round(avg_f1_score, 4)
summary_df = pd.DataFrame(summary.items(), columns=['Parameters', 'Value'])
db.put('dtn.model_summary', summary)
db.put('dtn.model_instance', tree)
except Exception as e:
traceback.print_exc()
return common.error_msg("Exception during training model: " + str(e))
div = html.Div([
html.H2('Model Parameters & Summary:'),
dbc.Table.from_dataframe(summary_df, striped=True, bordered=True, hover=True, style = common.table_style),
html.Br(),
html.H2('Tree'),
html.H2(str(tree)),
])
else:
div = common.error_msg('Select Proper Model Parameters!!')
return div
def dt_display_selected_file_scatter_plot(value):
db_value = db.get("dt.file")
if value is None and db_value is None:
return common.msg("Please select a cleaned file to proceed!!")
elif value is None and not db_value is None:
value = db_value
db.put("dt.file", value)
file = value
path = FileUtils.path('clean', file)
df = DataUtils.read_csv(path)
db.put("dt.data", df)
div = html.Div([
common.msg("Selected cleaned file: "+ file),
dbc.Table.from_dataframe(df.head(10).astype(str), striped=True, bordered=True, hover=True, style = common.table_style),
#html.Div([html.H3("Data Statistics")], style={'width': '100%', 'display': 'flex', 'align-items': 'center', 'justify-content': 'center'}),
#dbc.Table.from_dataframe(stats, striped=True, bordered=True, hover=True, style = common.table_style),
html.Br(),
get_dt_model_properties_div(df),
html.Div([], id = "dt-trained-model", style = {'margin': '10px'}),
])
return div
def upload_data(contents, filename):
"""Upload Files and Regenerate the file list."""
if contents:
for i in range(len(filename)):
FileUtils.upload(filename[i], contents[i])
return common.get_options('raw')
def nlcl_display_selected_file_scatter_plot(value):
db_value = db.get("nlcl.file")
if value is None and db_value is None:
return common.msg("Please select a cleaned file to proceed!!")
elif value is None and not db_value is None:
value = db_value
db.put("nlcl.file", value)
file = value
path = FileUtils.path('clean', file)
df = DataUtils.read_csv(path)
db.put("nlcl.data", df)
stats = df.describe(include = 'all').head(6).round(5)
stats.insert(loc=0, column='Statistics', value=['Count','unique','top','freq','Mean','Standard Deviation'])
stats = stats.drop(stats.index[[1,2,3]])
div = html.Div([
common.msg("Selected cleaned file: "+ file),
dbc.Table.from_dataframe(df.head(10), striped=True, bordered=True, hover=True, style = common.table_style),
html.Div([html.H3("Data Statistics")], style={'width': '100%', 'display': 'flex', 'align-items': 'center', 'justify-content': 'center'}),
dbc.Table.from_dataframe(stats, striped=True, bordered=True, hover=True, style = common.table_style),
html.Br(),
html.Div([html.H2("Scatter Plot")], style={'width': '100%', 'display': 'flex', 'align-items': 'center', 'justify-content': 'center'}),
dbc.Row([
dbc.Col([
dbc.Label("Select Class"),
dcc.Dropdown(
id = 'nlcl-class',
options=[{'label':col, 'value':col} for col in [*df]],
value=None,
multi=False
),
html.Br(),
dbc.Label("Select X Axis"),
dcc.Dropdown(
id = 'nlcl-x-axis',
options=[{'label':col, 'value':col} for col in [*df]],
value=None,
multi=False
),
html.Br(),
dbc.Label("Select Y Axis"),
dcc.Dropdown(
id = 'nlcl-y-axis',
options=[{'label':col, 'value':col} for col in [*df]],
value=None,
multi=False
),
html.Br(),
dbc.Button("Plot", color="primary", id = 'nlcl-scatter-plot-button'),
html.Div([], id = "nlcl-class-do-nothing"),
html.Div([], id = "nlcl-x-axis-do-nothing"),
html.Div([], id = "nlcl-y-axis-do-nothing")
], md=2,
style = {'margin': '10px', 'font-size': '16px'}),
dbc.Col([], md=9, id="nlcl-scatter-plot")
]),
html.Br(),
get_nlcl_model_properties_div(df),
html.Div([], id = "nlcl-trained-model", style = {'margin': '10px'}),
])
return div
#################################
# This has to be added to find the custom packages
import sys
import os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + '/../../')
#################################
import zipfile
from ml.neural_net.digit_recog_1_layer import DigitNeuralNet1HiddenLayer
from ml.framework.file_utils import FileUtils
train_data_zip_path = FileUtils.path('', 'mnist_train.csv.zip')
directory_to_extract_to = FileUtils.path('extra', '')
with zipfile.ZipFile(train_data_zip_path, 'r') as zip_ref:
zip_ref.extractall(directory_to_extract_to)
test_data_zip_path = FileUtils.path('', 'mnist_test.csv.zip')
with zipfile.ZipFile(test_data_zip_path, 'r') as zip_ref:
zip_ref.extractall(directory_to_extract_to)
#Neural Net Training
input_nodes = 784
hidden_nodes = 100
output_nodes = 10
learning_rate = 0.1
epoch = 5
train_data_path = FileUtils.path('extra', 'mnist_train.csv')
test_data_path = FileUtils.path('extra', 'mnist_test.csv')
"""App Main
python3 app_main
"""
import os
from ml.framework.file_utils import FileUtils
FileUtils.mkdir('raw')
FileUtils.mkdir('clean')
FileUtils.mkdir('knn')
from ml.ux import index
if __name__ == '__main__':
""
"""App Main
python3 app_main
"""
import os
from ml.framework.file_utils import FileUtils
FileUtils.mkdir('raw')
from ml.ux import index
if __name__ == '__main__':
""
'width': '50%',
'height': '50px',
'lineHeight': '50px',
'borderWidth': '1px',
'borderStyle': 'dashed',
'borderRadius': '5px',
'textAlign': 'center',
'margin': '10px'
},
multiple=True),
html.Br(),
html.Div([
html.H2("Select a file from all the uploaded files:"),
dcc.Dropdown(
id = 'selected-file',
options=[{'label':file, 'value':file} for file in FileUtils.files('raw')],
value=None,
multi=False
),
html.Br(),
],
style = {'margin': '10px', 'width': '50%'}),
html.Div([], id = "display-file"),
html.Div([], id = "file-properties"),
html.Div([], id = "file-separator-do-nothing"),
html.Div([], id = "file-header-do-nothing")
])
@app.callback(
Output("selected-file", "options"),
[Input('upload-data', 'contents'),
'lineHeight': '50px',
'borderWidth': '1px',
'borderStyle': 'dashed',
'borderRadius': '5px',
'textAlign': 'center',
'margin': '10px'
},
multiple=True),
html.Br(),
html.Div([
html.H2("Select a file from all the uploaded files:"),
dcc.Dropdown(id='selected-file',
options=[{
'label': file,
'value': file
} for file in FileUtils.files('raw')],
value=None,
multi=False),
html.Br(),
],
style={
'margin': '10px',
'width': '50%'
}),
html.Div([], id="display-file"),
html.Div([], id="file-properties"),
html.Div([], id="file-separator-do-nothing"),
html.Div([], id="file-header-do-nothing")
])
def get_options(dir: str):
files = FileUtils.files(dir)
if len(files) == 0:
return [{'label': 'No files yet!', 'value': 'None'}]
else:
return [{'label': file, 'value': file} for file in files]