def UnsupervisedClassification(DiskPath,filelist):
# Variables
#imdir = r'C:\CK\welldata-images\Disk1'
targetdir = os.getcwd()+"/"+ "Cluster/"+dirName
number_clusters = 4
featurelist = []
for i, imagepath in enumerate(filelist):
print(" Status: %s / %s" %(i, len(filelist)), end="\r")
img = image.load_img(imagepath,target_size=(224, 224))
img_data = image.img_to_array(img)
img_data = np.expand_dims(img_data, axis=0)
img_data = preprocess_input(img_data)
features = np.array(model.predict(img_data))
#features_flat = features.reshape(( -1,2))
featurelist.append(features.flatten())
# Clustering
kmeans = KMeans(n_clusters=number_clusters, random_state=0).fit(np.array(featurelist))
print('------------------------K-Means Clustering Completed------------------------')
try:
os.makedirs(targetdir)
except OSError:
pass
#Copy Image
print("\n")
for i, m in enumerate(kmeans.labels_):
print(" Copy: %s / %s" %(i, len(kmeans.labels_)), end="\r")
shutil.copy(filelist[i], targetdir + str(m) + "_" + str(i) + ".png")
#print('Before Upload Files')
uploadFile.UploadFiles('converge-clusterimages',DiskPath,str(m) + "_" + str(i) + ".png",targetdir + str(m) + "_" + str(i) + ".png")
4).value = SP
self.excel_sheet.cell(self.products_num + 2,
5).value = packing
print(self.products_num)
self.products_num += 1
except:
print("normal except")
continue
break
except:
continue
break
while True:
try:
self.wb.save(self.excel_path)
except:
print("Please check your excel file.")
continue
break
self.browser.close()
return self.products_num
if __name__ == "__main__":
ExtractProduct = ExtractProduct()
limit_row = ExtractProduct.get_information_from_each_product()
upload = UploadFile.UploadFile("JTC_Bot", limit_row, 7)
upload.uploadFile()
print("Task completed")
def ImageProcess(path, DiskPath):
try:
if ("Splits" in path):
outputDir = path.rsplit('/', 1)[0] + "\\Splits-Enhanced\\"
if (os.path.isdir(outputDir) == False):
os.makedirs(outputDir, exist_ok=True)
print('Created Enhanced images directory for splits')
else:
print(path)
outputDir = path.rsplit('\\', 1)[0] + "\\Enhanced\\"
if (os.path.isdir(outputDir) == False):
os.makedirs(outputDir, exist_ok=True)
print('Created Enhanced images directory')
file_name = path.rsplit('/', 1)[-1]
#print (os.path.join(root,filename) )
#file_name_withoutext=Constants.FileName
file_name_withoutext = path.rsplit('\\', 1)[1].split('.')[0]
im_new = process_image_for_ocr(path)
kernel = np.ones((1, 1), np.uint8)
closing = cv2.morphologyEx(im_new, cv2.MORPH_CLOSE, kernel)
#kernel = np.ones((1,1),np.uint8)
#erosion = cv2.erode(im_new,kernel,iterations = 50)
EnhancedImagePath = os.path.join(
outputDir, file_name_withoutext + '_Enhanced.png')
cv2.imwrite(EnhancedImagePath, closing)
#cv2.waitKey(0)
#fetch ContainerName From StorageContainer For ProcessedImages
#StorageAcountName=retrieveTableStorage.FetchValueFromTableStorage('ProcessedStorageAccountNameWest')
#import Constants as cons
#OriginLoc=cons.OriginLocation
#if(OriginLoc=='WH'):
#ProcessedSContainerName=retrieveTableStorage.FetchValueFromTableStorage('ProcessedStorageAccountNameWest')
#if(OriginLoc=='EH'):
#ProcessedSContainerName=retrieveTableStorage.FetchValueFromTableStorage('ProcessedStorageAccountNameEast')
#Comment this
#ProcessedSContainerName=Constants.ProcessedSContainerName
#StorageAcountName=retrieveTableStorage.FetchValueFromTableStorage('StorageAccountNameEast')
#uploadFiles.UploadFiles(ProcessedSContainerName,file_name_withoutext+'_Enhanced.png',os.path.join(outputDir,file_name_withoutext+'_Enhanced.png'))
uploadFile.UploadFiles('converge-enhancedimages', DiskPath,
file_name_withoutext + '_Enhanced.png',
EnhancedImagePath)
print('----------------Image Processing Completed for ' +
file_name_withoutext + '----------------')
#print('upload Done!')
#import Constants as cons
#OriginLoc=cons.OriginLocation
#UnComment Production
#FETCH STOAREGE ACCOUNT NAME
#if(OriginLoc=='WH'):
#StorageAcountName=retrieveTableStorage.FetchValueFromTableStorage('StorageAccountNameWest')
#if(OriginLoc=='EH'):
#StorageAcountName=retrieveTableStorage.FetchValueFromTableStorage('StorageAccountNameEast')
#UnComment Production
#StorageAcountName='029ze1b1storslbpcom'
# print('upload Done!!!')
#print('path:::'+path)
#path=path.replace("/", "\\")
#absPath=path.rsplit('\\', 1)[0]
#cwd=os.getcwd()
#fullpath="".join(absPath.rsplit(cwd))
#fullpath=fullpath.lstrip("\\")
#print('fullpath:::'+fullpath)
#fullpath=fullpath.replace("\\", "/")
#print('After fullpath:::'+fullpath)
#StorageAcountName = retrieveTableStorage.FetchValueFromTableStorage(Constants.UploadStorageAccountName)
#StorageAcountName='stgactssabackupdev'
#blobpath='https://'+StorageAcountName+'.blob.core.windows.net/'+ ProcessedSContainerName+'/'+fullpath+'/'+file_name_withoutext+'_Enhanced.png'
#insertintoCosmos.updateDocumentsinCosmos(file_name_withoutext+'.TIF','EnhancedImage_FilePath',blobpath)
return EnhancedImagePath
except Exception as e:
print('Error occurred in ImageProcess :::.', e)
def uploadFile():
uploaded_file = st.file_uploader("", type="csv")
if uploaded_file is not None:
dataFrame = pd.read_csv(uploaded_file)
labelencoder = LabelEncoder()
for col in dataFrame.columns:
dataFrame[col] = labelencoder.fit_transform(dataFrame[col])
try:
uploadedData = UploadFile.FileInfo(dataFrame)
st.success('Successfully uploaded!')
st.title('Dataset Information:')
dfInfo = st.selectbox("", [
'Head', 'Describe', 'Info', 'Isnull', 'Unique values and iteration'
])
if dfInfo == 'Head':
st.subheader('Dataframe head:')
st.write(dataFrame.head())
if dfInfo == 'Describe':
st.subheader('Dataframe description:')
st.write(dataFrame.describe())
if dfInfo == 'Info':
st.subheader('Dataframe informations:')
st.text(uploadedData.info())
if dfInfo == 'Isnull':
st.subheader('Null occurrences')
st.write(dataFrame.isnull().sum())
if dfInfo == 'Unique values and iteration':
col = st.selectbox('Choose a column to see unique values',
uploadedData.columns)
st.subheader('Unique values and iteration')
st.write(uploadedData.info2(col))
if st.sidebar.checkbox(" Train a Model ", False):
st.title('Choose an out put to Fit a model')
outputCol = st.selectbox('', uploadedData.columns)
colIndex = dataFrame.columns.get_loc(outputCol)
y = dataFrame.iloc[:, colIndex]
X = dataFrame.drop(columns=[outputCol])
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=0)
def plot_metrics(metrics_list):
if 'Confusion Matrix' in metrics_list:
st.subheader("Confusion Matrix")
plot_confusion_matrix(model,
X_test,
y_test,
display_labels=class_names)
st.pyplot()
if 'ROC Curve' in metrics_list:
st.subheader("ROC Curve")
plot_roc_curve(model, X_test, y_test)
st.pyplot()
if 'Precision-Recall Curve' in metrics_list:
st.subheader('Precision-Recall Curve')
plot_precision_recall_curve(model, X_test, y_test)
st.pyplot()
class_names = ['Case1', 'Case2']
st.sidebar.subheader("Choose Classifier")
classifier = st.sidebar.selectbox(
"", ("Support Vector Machine (SVM)", "Logistic Regression",
"Random Forest"))
if st.sidebar.checkbox("Feature Scaling", False):
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
if classifier == 'Support Vector Machine (SVM)':
st.sidebar.subheader("Model Hyperparameters")
C = st.sidebar.number_input("C (Regularization parameter)",
0.01,
10.0,
step=0.01,
key='C_SVM')
kernel = st.sidebar.radio("Kernel", ("rbf", "linear"),
key='kernel',
index=1)
gamma = st.sidebar.radio("Gamma (Kernel Coefficient)",
("scale", "auto"),
index=1,
key='gamma')
st.sidebar.subheader("Choose metrics to plot:")
metrics = st.sidebar.multiselect(
"", ('Confusion Matrix', 'ROC Curve',
'Precision-Recall Curve'))
if st.sidebar.button("Classify", key='classify'):
st.subheader("Support Vector Machine (SVM) Results")
model = SVC(C=C, kernel=kernel, gamma=gamma)
model.fit(X_train, y_train)
accuracy = model.score(X_test, y_test)
y_pred = model.predict(X_test)
st.write("Model accuracy: ", accuracy.round(2))
st.write(
"Model precision: ",
precision_score(y_test, y_pred,
labels=class_names).round(2))
st.write(
"Model recall: ",
recall_score(y_test, y_pred,
labels=class_names).round(2))
plot_metrics(metrics)
if classifier == 'Logistic Regression':
st.sidebar.subheader("Model Hyperparameters")
C = st.sidebar.number_input("C (Regularization parameter)",
0.01,
10.0,
step=0.01,
key='C_LR')
max_iter = st.sidebar.slider("Maximum number of iterations",
100,
500,
key='max_iter')
metrics = st.sidebar.multiselect(
"", ('Confusion Matrix', 'ROC Curve',
'Precision-Recall Curve'))
if st.sidebar.button("Classify", key='classify'):
st.subheader("Logistic Regression Results")
model = LogisticRegression(C=C,
penalty='l2',
max_iter=max_iter)
model.fit(X_train, y_train)
accuracy = model.score(X_test, y_test)
y_pred = model.predict(X_test)
st.write("Model accuracy: ", accuracy.round(2))
st.write(
"Model precision: ",
precision_score(y_test, y_pred,
labels=class_names).round(2))
st.write(
"Model recall: ",
recall_score(y_test, y_pred,
labels=class_names).round(2))
plot_metrics(metrics)
if classifier == 'Random Forest':
st.sidebar.subheader("Model Hyperparameters")
n_estimators = st.sidebar.number_input(
"The number of trees in the forest",
100,
5000,
step=10,
key='n_estimators')
max_depth = st.sidebar.number_input(
"The maximum depth of the tree",
1,
20,
step=1,
key='n_estimators')
bootstrap = st.sidebar.radio(
"Bootstrap samples when building trees", ('True', 'False'),
key='bootstrap')
metrics = st.sidebar.multiselect(
"", ('Confusion Matrix', 'ROC Curve',
'Precision-Recall Curve'))
if st.sidebar.button("Classify", key='classify'):
st.subheader("Random Forest Results")
model = RandomForestClassifier(n_estimators=n_estimators,
max_depth=max_depth,
bootstrap=bootstrap,
n_jobs=-1)
model.fit(X_train, y_train)
accuracy = model.score(X_test, y_test)
y_pred = model.predict(X_test)
st.write("Model accuracy: ", accuracy.round(2))
st.write(
"Model precision: ",
precision_score(y_test, y_pred,
labels=class_names).round(2))
st.write(
"Model recall: ",
recall_score(y_test, y_pred,
labels=class_names).round(2))
plot_metrics(metrics)
except:
st.error('Upload a CSV file to get started.')
self.excel_sheet.cell(self.products_num + 2, 1).value = url
self.excel_sheet.cell(self.products_num + 2, 2).value = name
self.excel_sheet.cell(self.products_num + 2, 3).value = category['category']
self.excel_sheet.cell(self.products_num + 2, 4).value = price
print(self.products_num)
self.products_num += 1
except:
print("normal except")
continue
break
print("--------------------------------------------")
while True:
try:
self.wb.save(self.excel_path)
except:
print("Please check your excel file.")
continue
break
self.browser.close()
return self.products_num
if __name__ == "__main__":
ExtractProduct = ExtractProduct()
ExtractProduct.get_all_categories()
limit_row = ExtractProduct.get_information_from_each_product()
upload = UploadFile.UploadFile("Paodeacucar", limit_row, 6)
upload.uploadFile()
print("Task completed")
self.excel_sheet.cell(self.products_num + 2, 5).value = priceperunit
self.excel_sheet.cell(self.products_num + 2, 6).value = amount
print(self.products_num)
self.products_num += 1
except:
print("normal except")
continue
break
except:
print("There is a issue from extracting each product url in the category : " + self.category_url[x]['category'])
continue
break
while True:
try:
self.wb.save(self.excel_path)
except:
print("Please check your excel file.")
continue
break
self.browser.close()
return self.products_num
if __name__ == "__main__":
ExtractProduct = ExtractProduct()
limit_row = ExtractProduct.extract_each_product_url_and_save()
upload = UploadFile.UploadFile("ProductFromShop", limit_row, 8)
upload.uploadFile()
print("Task completed")
url_index += 1
except:
print("normal except")
continue
break
except:
print("normal except_2")
continue
break
while True:
try:
self.wb.save(self.excel_path)
except:
print("Please check your excel file.")
continue
break
self.browser.close()
return url_index
if __name__ == "__main__":
ExtractProduct = ExtractProduct()
ExtractProduct.get_total_num_of_products()
ExtractProduct.get_url_of_each_product()
limit_row = ExtractProduct.get_information_from_each_product()
upload = UploadFile.UploadFile("MultifoodsBOT", limit_row, 12)
upload.uploadFile()
print("Task completed")
except:
continue
break
try:
self.browser.find_element_by_class_name(
'next.i-next').click()
except:
finished = True
continue
while True:
try:
self.wb.save(self.excel_path)
except:
print("Please check your excel file.")
continue
break
self.browser.close()
return self.products_num
if __name__ == "__main__":
ExtractProduct = ExtractProduct()
ExtractProduct.login()
ExtractProduct.get_categories()
limit_row = ExtractProduct.get_info_of_each_product()
upload = UploadFile.UploadFile("ParceiroAmbev", limit_row, 8)
upload.uploadFile()
print("Task completed")
self.excel_sheet.cell(
i + 2, 3).value = self.products_url[i]['category']
self.excel_sheet.cell(i + 2, 4).value = price
self.excel_sheet.cell(i + 2, 5).value = package_type
print(str(self.products_num) + "/" + str(i))
except:
print("normal except")
continue
break
while True:
try:
self.wb.save(self.excel_path)
except:
print("Please check your excel file.")
continue
break
self.browser.close()
return self.products_num + 1
if __name__ == "__main__":
ExtractProduct = ExtractProduct()
ExtractProduct.get_all_products_url()
limit_row = ExtractProduct.get_information_from_each_product()
upload = UploadFile.UploadFile("Vinhais", limit_row, 7)
upload.uploadFile()
print("Task completed")
import sys
from requests import HTTPError
import UploadFile
import UserInterface
if __name__ == '__main__':
try:
class_id = UserInterface.get_course()
assign_id = UserInterface.get_assignment(class_id)
file_name = UserInterface.get_file()
UploadFile.upload_file(class_id, assign_id, file_name)
except HTTPError:
print("Canvas returned an error. Please try again later.", file=sys.stderr)
sys.exit()
print("\nAssignment submitted.")
5).value = amount
print(self.products_num)
self.products_num += 1
self.browser.find_element_by_class_name(
'ant-modal-close').click()
time.sleep(1)
except:
print("normal except")
continue
break
while True:
try:
self.wb.save(self.excel_path)
except:
print("Please check your excel file.")
continue
break
self.browser.close()
return self.products_num
if __name__ == "__main__":
ExtractProduct = ExtractProduct()
limit_rows = ExtractProduct.get_information_from_each_product()
upload = UploadFile.UploadFile("Frubana", limit_rows, 7)
upload.uploadFile()
print("Task completed")
self.products_num + 2,
5).value = price_unit
print(self.products_num)
self.products_num += 1
except:
print("normal except")
continue
break
except:
continue
break
while True:
try:
self.wb.save(self.excel_path)
except:
print("Please check your excel file.")
continue
break
self.browser.close()
return self.products_num
if __name__ == "__main__":
ExtractProduct = ExtractProduct()
ExtractProduct.login()
limit_row = ExtractProduct.get_information_from_each_product()
upload = UploadFile.UploadFile("PORCO FELIZ", limit_row, 7)
upload.uploadFile()
print("Task completed")
break
while True:
try:
# self.browser.execute_script("window.history.go(-1)")
self.browser.back()
time.sleep(2)
except:
print("back error")
continue
break
while True:
try:
self.wb.save(self.excel_path)
except:
print("Please check your excel file.")
continue
break
self.browser.close()
return self.products_num
if __name__ == "__main__":
ExtractProduct = ExtractProduct()
limit_row = ExtractProduct.get_information_from_each_product()
upload = UploadFile.UploadFile("EstoqueOnlineBOT", limit_row, 7)
upload.uploadFile()
print("Task completed")
