def main():
dir_name = 'Audio_Speech_Actors_01-24-2/'
base_filename = 'Actor_'
names = 'Actor'
# define the address of the dataset
file_diagram = [dir_name + base_filename + str(i) + '/' + names + str(i) + '_' + str(j) + '.wav'
for i, j in itertools.product(range(1, 25), range(5, 61))]
d = pd.DataFrame({'Entropy': [], 'Duracion': [], 'Tempo': []})
diagramas = []
for item in file_diagram:
#using function emotions to get the embedder and duration in seconds of audio
embedder, dur_sec, tempo, yk = emotions(item)
signal = emotion_second(item)
entropy, diagramas = get_diag(yk, diagramas)
d = d.append({'Entropy': float(entropy[0]), 'Duracion': dur_sec, 'Tempo': tempo}, ignore_index=True)
print(item)
machine_learning(d, diagramas)
fig, ax = plt.subplots(nrows=3, ncols=1, figsize=(15, 14))
# plotear la serie de tiempo y la embedida
ax[0].plot(Signal)
ax[1].plot(embedder[0, :], embedder[1, :])
ax[2].plot(Signal)
# plot the 3D embedding
ax = fig.add_subplot(3, 1, 3, projection='3d')
ax.plot(embedder3[0, :], embedder3[1, :], embedder3[2, :])
return
def work_flow():
# load data that will be used in the script
dir_name = 'Audio_Speech_Actors_01-24-2/'
base_filename = 'Actor_'
names = 'Actor'
# define the address of the dataset
file = [
dir_name + base_filename + str(i) + '/' + names + str(i) + '_' +
str(j) + '.wav'
for i, j in itertools.product(range(1, 25), range(1, 61))
]
#initializate the dataframe to fill
d = pd.DataFrame({'Entropy': [], 'Duracion': [], 'Tempo': []})
diagramas = []
for item in file:
#using function emotions to get the embedder and duration in seconds of audio
try:
embedder, dur_sec, tempo = emotions(item)
#get diagrams
entropy, diagramas = get_diag(embedder, diagramas)
d = d.append(
{
'Entropy': float(entropy[0]),
'Duracion': dur_sec,
'Tempo': tempo
},
ignore_index=True)
print(item)
except:
pass
#plot(embedder)
machine_learning(d, diagramas)
return
def post(self):
# read information from csv file
df = pd.read_csv("user_houses.csv",
usecols=[
"Identifier", "UserID", "Distance", "Bedroom",
"Bathroom", "Car", "Landsize", "BuildingArea",
"YearBuilt", "Lattitude", "Longtitude", "Suburb",
"Street", "Type", "Regionname", "Predicted_Price"
])
df.set_index(["Identifier"], inplace=True)
print("###################df####################")
print(df.to_string())
# since post new data for predict price, so accessing dataset here is test dataset
# for user to provide a new house and its details into dataset
house = request.json # the content that post request provide
df_log = pd.read_csv("log_file.csv",
usecols=["ID", "UserID", "Operation", "Time"])
df_log.set_index(["ID"], inplace=True)
print(df_log.to_string())
get_time = time.asctime(time.localtime(time.time()))
if df_log.empty:
last_index = 0
else:
# last_file = df_log.tail(1)
# print(int(last_file["ID"].values))
# print(last_file.index)
last_index = df_log.index[-1] + 1
logID = last_index
# df_log.loc[logID, "ID"] = last_index
df_log.loc[logID, "UserID"] = house["UserID"]
df_log.loc[logID, "Operation"] = "Predict"
df_log.loc[logID, "Time"] = get_time
print(df_log.to_string())
df_log.to_csv("log_file.csv")
if "Identifier" not in house:
return {"message": "Identifier is missing."}, 400
id = house[
"Identifier"] # id is the Identifier of the house which a user just post
# check the Identifier whether already exist
if id in df.index:
return {
"message":
"Sorry, A house with identifier {} is already exist, please create a new identifier."
.format(id)
}, 400
# put the value into the dataset
df_predicted = pd.DataFrame(columns=[
"Identifier", "UserID", "Distance", "Bedroom", "Bathroom", "Car",
"Landsize", "BuildingArea", "YearBuilt", "Lattitude", "Longtitude",
"Suburb", "Street", "Type", "Regionname"
])
# print("在预测price之前的df;")
print(df.to_string())
for key in house:
if str(key) == "Identifier":
continue
if key not in house_model.keys():
# unexpected features
return {
"message": "Property {} is invalid.".format(str(key))
}, 400
if key == "Suburb": # whe suburb part, check the input string whether valid or not
if house[key] not in suburb_list:
return {
"message":
"Property {}'s value is invalid".format(str(key))
}, 400
if key == "Street":
if house[key] not in street_list:
return {
"message":
"Property {}'s value is invalid".format(str(key))
}, 400
if key == "Type":
if house[key] not in type_list:
return {
"message":
"Property {}'s value is invalid".format(str(key))
}, 400
if key == "Regionname":
if house[key] not in region_list:
return {
"message":
"Property {}'s value is invalid".format(str(key))
}, 400
df_predicted.loc[0, key] = house[key]
df.loc[id, key] = house[key]
# recall the regression function to predict the new dataset, and return the dataset with predict price column
# remove the user and password columns and send to the subset part to machine learning function
# df1 = df.query("Identifer == @id")
# only get a specific row information to predict price
# print("在预测price之前的df;")
print(df.to_string())
# 这里调用 machine_learning 的信息可能还是用问题，主要是好想表格的输入输出类型不一致。
df1 = ml.data_processing(df_predicted)
result = ml.machine_learning(df1) # result has price and index only
print("*************result********************")
print(result.to_string())
price = result.loc[0, "Price"]
df.loc[id, "Predicted_Price"] = price
print("##############final df is ##############")
print(df.to_string())
df.to_csv('user_houses.csv')
# 样品输出
return {
"HouseID":
id,
"price":
price,
"message":
"The house {} information has been posted and predict price is {}".
format(id, price)
}, 200
def put(self, id):
df = pd.read_csv("user_houses.csv",
usecols=[
"Identifier", "UserID", "Distance", "Bedroom",
"Bathroom", "Car", "Landsize", "BuildingArea",
"YearBuilt", "Lattitude", "Longtitude", "Suburb",
"Street", "Type", "Regionname", "Predicted_Price"
])
df.set_index(["Identifier"], inplace=True)
print(df.to_string())
print(df.index)
# the accessing dataset is test dataset
args = parser.parse_args()
# 我觉得可能不需要user和password作为query parameter了，因为expect model里面有user 和password
user = float(args.get("UserID"))
# password = args.get("Password")
print(id)
print(df.index)
if id not in df.index:
print("************************************************")
api.abort(404, "House with id {} is not exist.".format(id))
house = request.json # covert the request to json
df_log = pd.read_csv("log_file.csv",
usecols=["ID", "UserID", "Operation", "Time"])
df_log.set_index(["ID"], inplace=True)
print(df_log.to_string())
get_time = time.asctime(time.localtime(time.time()))
if df_log.empty:
last_index = 0
else:
# last_file = df_log.tail(1)
# print(int(last_file["ID"].values))
# print(last_file.index)
last_index = df_log.index[-1] + 1
logID = last_index
#df_log.loc[logID, "ID"] = last_index
df_log.loc[logID, "UserID"] = int(df.loc[id, "UserID"])
df_log.loc[logID, "Operation"] = "Update house Info"
df_log.loc[logID, "Time"] = get_time
print(df_log.to_string())
df_log.to_csv("log_file.csv")
# for checking the user whether has the right to access the house information
print("################User ID ###############")
print(df.loc[id, "UserID"])
if df.loc[id, "UserID"] - user != 0:
api.abort(
403,
"message: Sorry, you has no right to access house {}".format(
id))
#
# if user is not df.loc[id, "UserID"]:
# abort(400, message="No permission to access the house information")
#print("没有更新房子信息之前的：")
print(df.to_string())
if "Identifier" in house and id != house["Identifier"]:
return {"message": "House Identifier cannot be changed."}, 400
df_predicted = pd.DataFrame(columns=[
"Identifier", "UserID", "Distance", "Bedroom", "Bathroom", "Car",
"Landsize", "BuildingArea", "YearBuilt", "Lattitude", "Longtitude",
"Suburb", "Street", "Type", "Regionname"
])
# Update the house information
# Update the house information
for key in house:
if str(key) == "Identifier":
continue
if key not in house_model.keys():
return {
"message": "Property {} is invalid.".format(str(key))
}, 400
if key == "Suburb": # whe suburb part, check the input string whether valid or not
if house[key] not in suburb_list:
return {
"message":
"Property {}'s value is invalid".format(str(key))
}, 400
if key == "Street":
if house[key] not in street_list:
return {
"message":
"Property {}'s value is invalid".format(str(key))
}, 400
if key == "Type":
if house[key] not in type_list:
return {
"message":
"Property \"{}\" value is invalid :\"{}\" ".format(
str(key), house[key])
}, 400
if key == "Regionname":
if house[key] not in region_list:
return {
"message":
"Property {}'s value is invalid".format(str(key))
}, 400
df.loc[id, key] = house[key]
if str(key) != "Price":
df_predicted.loc[0, key] = house[key]
# print("更新房子信息之后的：")
print(df.to_string())
# recall the regression function to predict price
# price = regression_function(df).loc[id, "Price"]
# price = df_price.loc[id,"Price"]
# remove the user and password columns and send to the subset part to machine learning function
# remove the user and password columns and send to the subset part to machine learning function
print("##############df_predicted#############")
print(df_predicted.to_string())
df1 = ml.data_processing(df_predicted)
result = ml.machine_learning(df1) # result has price and index only
# df.loc[id, "Price"] = result[id, "Price"] # update the houses information in csv file
# df.to_csv("user_houses.csv")
print("#############reuslt####################")
print(result.to_string())
price = result.loc[0, "Price"]
df.loc[id, "Predicted_Price"] = price
print("############final df#############")
print(df.to_string())
df.to_csv('user_houses.csv')
# return price, 200
return {
"message":
"House {} information has been updated and predict price {}".
format(id, price)
}, 200
def page3():
train_file_list = os.listdir(train_file_path)
test_file_list = os.listdir(test_file_path)
embed_model_list = os.listdir(embed_model_path)
machine_model_list = os.listdir(machine_model_path)
if request.method == "POST":
# 시각화 버튼을 눌렀을 경우
if request.form.get("visual_button"):
response_data = request.form.get("visual_button")
response_data = json.loads(response_data)
print(response_data)
trainFile = response_data['trainData']
testFile = response_data['testData']
# 데이터 읽기
train = pd.read_csv(train_file_path + trainFile)
test = pd.read_csv(test_file_path + testFile)
# 결측치가 있는지 확인하기(우선은 제거하는 방식)
if pd.isnull(train['x']).sum() > 0 or pd.isnull(
train['y']).sum() > 0:
train = train.dropna()
if pd.isnull(test['x']).sum() > 0 or pd.isnull(
test['y']).sum() > 0:
test = test.dropna()
train = train.sample(frac=1).reset_index(drop=True)
test = test.sample(frac=1).reset_index(drop=True)
# 1) 처음 임베딩 및 시각화인 경우 -> 임베딩 파라미터만 받아오면 됨
# is_pre_embed 없음, is_pre_train 없음, machine_value []
if 'is_pre_embed' not in response_data and 'is_pre_machine' not in response_data and response_data[
'machine_value'] == []:
print('first-embed, no-machine')
embed_type = response_data['embed_type']
embed_params = get_embed_params(embed_type,
response_data['embed_value'])
# 임베딩
X_train, X_test, y_train, y_test = embedding(
trainFile.split(".")[0], embed_type, train, test,
embed_params)
# 차원축소
dimension_type = response_data['dimension_type']
dimension_reduction(dimension_type, X_train, X_test, y_train,
y_test)
return render_template(
'visualization.html',
visualization="embedding_and_visualization")
# 2) pre 임베딩 및 시각화인 경우 -> 어떠한 파라미터도 받을 필요 없음
# is_pre_embed 있음, is_pre_train 없음, embed_value [], machine_value []
elif 'is_pre_embed' in response_data and 'is_pre_machine' not in response_data and response_data[
'embed_value'] == [] and response_data[
'machine_value'] == []:
print('pre-embed, no-machine')
embed_type = response_data['embed_type']
pre_embed_model = response_data['pre_embed_model']
# 임베딩
X_train, X_test, y_train, y_test = pre_train_embedding(
embed_type, pre_embed_model, train, test)
# 차원축소
dimension_type = response_data['dimension_type']
dimension_reduction(dimension_type, X_train, X_test, y_train,
y_test)
return render_template(
'visualization.html',
visualization="embedding_and_visualization")
# 3) 처음 임베딩 및 처음 머신러닝 및 시각화인 경우 -> 임베딩, 머신러닝 파라미터 모두 받아오면 됨
# is_pre_embed 없음, is_pre_train 없음, machine_value 있음
elif 'is_pre_embed' not in response_data and 'is_pre_machine' not in response_data and response_data[
'machine_value'] != []:
print('first-embed, first-machine')
embed_type = response_data['embed_type']
embed_params = get_embed_params(embed_type,
response_data['embed_value'])
machine_type = response_data['machine_type']
machine_params = get_machine_params(
machine_type, response_data['machine_value'])
# 임베딩
X_train, X_test, y_train, y_test = embedding(
trainFile.split(".")[0], embed_type, train, test,
embed_params)
# 차원축소
dimension_type = response_data['dimension_type']
dimension_reduction(dimension_type, X_train, X_test, y_train,
y_test)
# 머신러닝
train_y_pred, test_y_pred = machine_learning(
embed_type, machine_type, X_train, X_test, y_train, y_test,
machine_params)
# 4) pre 임베딩 및 처음 머신러닝 및 시각화인 경우 -> 머신러닝 파라미터만 받아오면 됨
# is_pre_embed 있음, is_pre_train 없음, embed_value [], machine_value 있음
elif 'is_pre_embed' in response_data and 'is_pre_machine' not in response_data and response_data[
'embed_value'] == [] and response_data[
'machine_value'] != []:
print('pre-embed, first-machine')
embed_type = response_data['embed_type']
pre_embed_model = response_data['pre_embed_model']
machine_type = response_data['machine_type']
machine_params = get_machine_params(
machine_type, response_data['machine_value'])
# 임베딩
X_train, X_test, y_train, y_test = pre_train_embedding(
embed_type, pre_embed_model, train, test)
# 차원축소
dimension_type = response_data['dimension_type']
dimension_reduction(dimension_type, X_train, X_test, y_train,
y_test)
# 머신러닝
train_y_pred, test_y_pred = machine_learning(
embed_type, machine_type, X_train, X_test, y_train, y_test,
machine_params)
# 5) pre 임베딩 및 pre 머신러닝 및 시각화인 경우 -> 어떠한 파라미터도 받을 필요 없음
# is_pre_embed 있음, is_pre_train 있음
elif 'is_pre_embed' in response_data and 'is_pre_machine' in response_data:
print('pre-embed, pre-machine')
embed_type = response_data['embed_type']
machine_type = response_data['machine_type']
pre_embed_model = response_data['pre_embed_model']
# 임베딩
X_train, X_test, y_train, y_test = pre_train_embedding(
embed_type, pre_embed_model, train, test)
# 차원축소
dimension_type = response_data['dimension_type']
dimension_reduction(dimension_type, X_train, X_test, y_train,
y_test)
# 머신러닝
train_y_pred, test_y_pred = pre_train_machine_learning(
embed_type, machine_type, X_train, X_test, y_train, y_test)
# 훈련 종료 후 머신러닝 결과
from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, f1_score
target_names = list(set(y_train))
train_df = pd.DataFrame(confusion_matrix(y_train, train_y_pred),
index=target_names,
columns=target_names)
test_df = pd.DataFrame(confusion_matrix(y_test, test_y_pred),
index=target_names,
columns=target_names)
path = r'/home/ubuntu/project2/csv_files/'
train_df.to_csv(path + 'confusion_matrix_train.csv', index=False)
test_df.to_csv(path + 'confusion_matrix_test.csv', index=False)
# 분류 평가 지표
train_accuracy = accuracy_score(y_train, train_y_pred)
train_precision = precision_score(y_train,
train_y_pred,
average='macro')
train_recall = recall_score(y_train, train_y_pred, average='macro')
train_f1 = f1_score(y_train, train_y_pred, average='macro')
test_accuracy = accuracy_score(y_test, test_y_pred)
test_precision = precision_score(y_test,
test_y_pred,
average='macro')
test_recall = recall_score(y_test, test_y_pred, average='macro')
test_f1 = f1_score(y_test, test_y_pred, average='macro')
print('train accuracy: {}, test accuracy: {}'.format(
train_accuracy, test_accuracy))
print('train precision: {}, test precision: {}'.format(
train_precision, test_precision))
print('train recall: {}, test recall: {}'.format(
train_recall, test_recall))
print('train f1: {}, test f1: {}'.format(train_f1, test_f1))
train_score_df = pd.DataFrame(columns=['Metrics', 'Score'])
train_score_df['Metrics'] = [
'accuracy', 'precision', 'recall', 'f1'
]
train_score_df['Score'] = [
round(train_accuracy, 2),
round(train_precision, 2),
round(train_recall, 2),
round(train_f1, 2)
]
train_score_df.to_csv(path + 'metrics_score_train.csv',
index=False)
test_score_df = pd.DataFrame(columns=['Metrics', 'Score'])
test_score_df['Metrics'] = [
'accuracy', 'precision', 'recall', 'f1'
]
test_score_df['Score'] = [
round(test_accuracy, 2),
round(test_precision, 2),
round(test_recall, 2),
round(test_f1, 2)
]
test_score_df.to_csv(path + 'metrics_score_test.csv', index=False)
train_df = pd.read_csv(path +
'embedding_and_visualization_train.csv')
test_df = pd.read_csv(path +
'embedding_and_visualization_test.csv')
train_df['pred'] = train_y_pred
train_df['success'] = train_df['pred'] == train_df['target']
train_df['success'] = train_df['success'].astype(int)
test_df['pred'] = test_y_pred
test_df['success'] = test_df['pred'] == test_df['target']
test_df['success'] = test_df['success'].astype(int)
success_mapping_table = {0: "실패", 1: "성공"}
train_df['success'] = train_df['success'].map(
success_mapping_table)
test_df['success'] = test_df['success'].map(success_mapping_table)
train_df.to_csv(
path + 'embedding_and_machinelearning_visualization_train.csv',
index=False)
test_df.to_csv(
path + 'embedding_and_machinelearning_visualization_test.csv',
index=False)
return render_template(
'visualization.html',
visualization="embedding_and_machineLearning_visualization")
return render_template('machineLearning.html',
train_file_list=train_file_list,
test_file_list=test_file_list,
embed_model_list=embed_model_list,
machine_model_list=machine_model_list)
else:
return render_template("machineLearning.html",
train_file_list=train_file_list,
test_file_list=test_file_list,
embed_model_list=embed_model_list,
machine_model_list=machine_model_list)