def splitimage(mydir):
print("清空資料夾.....")
deletefile(config_toPath)
for path, subdirs, files in os.walk(mydir):
for name in files:
imagename = name.replace(".jpg", "")
imagenumber = int(imagename.split("_")[1])
basedir = os.path.basename(path)
if imagenumber in datatest:
shutil.copy(
path + "\\" + name,
"C:\\Users\\409LAB00\\Desktop\\getRandomImageTest\\test\\"
+ basedir + "\\" + name)
print("copy " + name + " to testing dataset ")
elif imagenumber in datatraining:
shutil.copy(
path + "\\" + name,
"C:\\Users\\409LAB00\\Desktop\\getRandomImageTest\\train\\"
+ basedir + "\\" + name)
print("copy " + name + " to training dataset ")
elif imagenumber in datavalidation:
shutil.copy(
path + "\\" + name,
"C:\\Users\\409LAB00\\Desktop\\getRandomImageTest\\validation\\"
+ basedir + "\\" + name)
print("copy " + name + " to validation dataset ")
print("training set : ", datatraining)
print("validation set : ", datavalidation)
print("test set : ", datatest)
def test_decompressZip_to_uploadGeojsonFilesToPostgresTable(self):
zippath = "C:/Users/jeisonle/repository/Car-Routing/DigiroadPreDataAnalysis/digiroad/test/data/geojson/Subsets/subset1/summary1.zip"
outputFolder = self.dir + "%data%outputData%summaries".replace(
"%", os.sep)
outputFolder = os.path.join(outputFolder,
os.path.basename(zippath).split(".")[-2])
self.fileActions.decompressZipfile(zippath, outputFolder)
columns = {
"startPoint_YKR_ID": "ykr_from_id",
"endPoint_YKR_ID": "ykr_to_id",
"total_travel_time": "travel_time",
}
tableName = "cardat_fast_" + self.bicycleTravelTimeMatrix + ""
for root, dirs, files in os.walk(outputFolder):
for filename in files:
if filename.endswith("geojson"):
f = os.path.join(root, filename)
print("Loading: %s" % filename)
isExecuted = self.spatialPatter.insertData(
f, tableName, columns)
if isExecuted:
os.remove(f)
self.assertTrue(isExecuted)
def get_format_data(problem_title):
path = problem_save_path + "\\" + problem_title
for root, dirs, files in os.walk(path):
# 遍历文件
data_dict = {}
for file_name in files:
with open(os.path.join(root, file_name), 'r') as f:
txt = f.read()
if re.search(r'output\d+.txt', f.name):
# print("match")
txt = remove_blank_chars(txt)
# print(txt)
data_dict[file_name] = txt
data_total = len(data_dict)
# print(data_total)
data_list = []
for i in range(1, data_total // 2 + 1):
data = []
for s in ["input", "output"]:
name = s + str(i) + ".txt"
try:
data.append(data_dict[name])
# print("data_list add:{}".format(name))
except KeyError:
print("file_error... jumping：{}".format(path + file_name))
return None
data_list.append(data)
# 解析成功
else:
return data_list
def deletefile(mydir):
for path, subdirs, files in os.walk(mydir):
for name in files:
basedir = os.path.basename(path)
newpath = path.replace("\\" + basedir, '')
typeofdatadir = os.path.basename(newpath)
os.remove(mydir + "\\" + typeofdatadir + "\\" + basedir + "\\" +
name)
def deleteTrainingDataFirstTime(dir):
for path, subdirs, files in os.walk(dir):
basename = os.path.basename(path)
if (basename == "TrainingDataSet"):
print()
else:
shutil.rmtree(dir + "\\" + basename, ignore_errors=True)
print("刪除資料夾與檔案")
print("TrainingDataSetDir:", basename)
def deletefile(mydir):
for path, subdirs, files in os.walk(mydir):
for name in files:
basedir = os.path.basename(path)
newpath = path.replace("\\" + basedir, '')
typeofdatadir = os.path.basename(newpath)
if (typeofdatadir == 'getRandomImageTest'):
os.remove(mydir + '\\test\\' + name)
else:
os.remove(mydir + "\\" + typeofdatadir + "\\" + basedir +
"\\" + name)
def main():
for root, dirs, files in os.walk(base_save_path):
# 遍历所有的文件夹
for d in dirs:
# print(os.path.join(root, d))
title = d
query = {PROBLEM.TITLE: title}
problem = problem_table.find_one(query)
print(problem)
if problem[PROBLEM.STATE] == STATE_VALUE.FILE_SUCCESS or problem[
PROBLEM.STATE] == STATE_VALUE.DATA_SUCCESS:
# print(STATE_VALUE.FILE_SUCCESS)
save_file_to_db(query, problem)
def save_file_to_db(query, problem):
"""
data format : [[in,out]...]
:param problem:
:return:
"""
path = base_save_path + "\\" + problem[PROBLEM.TITLE]
for root, dirs, files in os.walk(path):
# 遍历文件
data_dict = {}
for file_name in files:
with open(os.path.join(root, file_name), 'r') as f:
txt = f.read()
# print(txt)
data_dict[file_name] = txt
data_total = len(data_dict)
# print(data_total)
data_list = []
for i in range(1, data_total // 2 + 1):
data = []
for s in ["input", "output"]:
name = s + str(i) + ".txt"
try:
data.append(data_dict[name])
# print("data_list add:{}".format(name))
except KeyError:
problem[PROBLEM.STATE] = STATE_VALUE.PARSE_DATA_ERROR
print("file_error... jumping：{}".format(path + file_name))
return
data_list.append(data)
# 解析成功
else:
# print(data_dict)
# data save in db is a json string
data_json = json.dumps(data_list)
problem[PROBLEM.DATA] = data_json
# print(data_json)
problem[PROBLEM.STATE] = STATE_VALUE.DATA_SUCCESS
# print(problem)
new_problem = {"$set": problem}
try:
problem_table.find_one_and_update(query, new_problem)
print(problem)
except Exception:
traceback.print_exc()
problem[PROBLEM.DATA] = ""
problem[PROBLEM.STATE] = STATE_VALUE.PARSE_DATA_ERROR
print("judge_data to big!!!!!: {}".format(name))
problem_table.find_one_and_update(query, new_problem)
def main():
for root, dirs, files in os.walk(problem_save_path):
# 遍历所有的文件夹
for d in dirs:
# print(os.path.join(root, d))
title = d
query = {Problem.TITLE: title}
problem = problem_collection.find_one(query)
# print(problem)
try:
if problem[Problem.DATA_STATUS] == StateValue.FILE_SUCCESS or problem[
Problem.DATA_STATUS] == StateValue.DATA_SUCCESS:
save_file_to_db(query, problem)
except Exception:
print(problem)
traceback.print_exc()
def absoluteFilePaths(directory):
for dirpath, _, filenames in os.walk(directory):
for f in filenames:
yield os.path.abspath(os.path.join(dirpath, f))
def splitimage(mydir):
print("清空資料夾.....")
deletefile(config_toPath)
for path, subdirs, files in os.walk(mydir):
basename = os.path.basename(path)
if (basename in batikset):
for name in files:
imagename = name.replace(".jpg", "")
imagenumber = int(imagename.split("_")[1])
basedir = os.path.basename(path)
if imagenumber in datatest:
shutil.copy(path + "\\" + name,
config_toPath + "\\test\\" + name)
if (oooos.path.exists(
"D:\\Maimy\\CNN\\TestDataSet\\batchTime" +
batchtime)):
if (oooos.path.exists(
"D:\\Maimy\\CNN\\TestDataSet\\batchTime" +
batchtime + "\\test" + montime)):
print("存在test" + montime + "資料夾")
else:
print("建立資料夾")
oooos.makedirs(
"D:\\Maimy\\CNN\\TestDataSet\\batchTime" +
batchtime + "\\test" + montime)
else:
oooos.makedirs(
"D:\\Maimy\\CNN\\TestDataSet\\batchTime" +
batchtime)
if (oooos.path.exists(
"D:\\Maimy\\CNN\\TestDataSet\\batchTime" +
batchtime + "\\test" + montime)):
print("存在test" + montime + "資料夾")
else:
print("建立資料夾")
oooos.makedirs(
"D:\\Maimy\\CNN\\TestDataSet\\batchTime" +
batchtime + "\\test" + montime)
shutil.copy(
path + "\\" + name,
"D:\\Maimy\\CNN\\TestDataSet\\batchTime" + batchtime +
"\\test" + montime + "\\" + name)
print("copy " + name + " to testing dataset ")
elif imagenumber in datatraining:
shutil.copy(
path + "\\" + name,
config_toPath + "\\train\\" + basedir + "\\" + name)
if (oooos.path.exists(
"D:\\Maimy\\CNN\\TrainingDataSet\\batchTime" +
batchtime)):
if (oooos.path.exists(
"D:\\Maimy\\CNN\\TrainingDataSet\\batchTime" +
batchtime + "\\training" + montime)):
print("存在training" + montime + "資料夾")
else:
print("建立資料夾")
oooos.makedirs(
"D:\\Maimy\\CNN\\TrainingDataSet\\batchTime" +
batchtime + "\\training" + montime)
else:
oooos.makedirs(
"D:\\Maimy\\CNN\\TrainingDataSet\\batchTime" +
batchtime)
if (oooos.path.exists(
"D:\\Maimy\\CNN\\TrainingDataSet\\batchTime" +
batchtime + "\\training" + montime)):
print("存在training" + montime + "資料夾")
else:
print("建立資料夾")
oooos.makedirs(
"D:\\Maimy\\CNN\\TrainingDataSet\\batchTime" +
batchtime + "\\training" + montime)
shutil.copy(
path + "\\" + name,
"D:\\Maimy\\CNN\\TrainingDataSet\\batchTime" +
batchtime + "\\training" + montime + "\\" + name)
print("copy " + name + " to training dataset ")
elif imagenumber in datavalidation:
shutil.copy(
path + "\\" + name, config_toPath + "\\validation\\" +
basedir + "\\" + name)
if (oooos.path.exists(
"D:\\Maimy\\CNN\\ValidationDataSet\\batchTime" +
batchtime)):
if (oooos.path.exists(
"D:\\Maimy\\CNN\\ValidationDataSet\\batchTime"
+ batchtime + "\\validation" + montime)):
print("存在validation" + montime + "資料夾")
else:
print("建立資料夾")
oooos.makedirs(
"D:\\Maimy\\CNN\\ValidationDataSet\\batchTime"
+ batchtime + "\\validation" + montime)
else:
oooos.makedirs(
"D:\\Maimy\\CNN\\ValidationDataSet\\batchTime" +
batchtime)
if (oooos.path.exists(
"D:\\Maimy\\CNN\\ValidationDataSet\\batchTime"
+ batchtime + "\\validation" + montime)):
print("存在validation" + montime + "資料夾")
else:
print("建立資料夾")
oooos.makedirs(
"D:\\Maimy\\CNN\\ValidationDataSet\\batchTime"
+ batchtime + "\\validation" + montime)
shutil.copy(
path + "\\" + name,
"D:\\Maimy\\CNN\\ValidationDataSet\\batchTime" +
batchtime + "\\validation" + montime + "\\" + name)
print("copy " + name + " to validation dataset ")
print("training set : ", datatraining)
print("validation set : ", datavalidation)
print("test set : ", datatest)
print("training set : ", len(datatraining))
print("validation set : ", len(datavalidation))
print("test set : ", len(datatest))
import random as r
from RandomImageConfig import *
from PIL import Image
from gevent import os
import os as oooos
import shutil
import sys
from os import system
count = 0
for path, subdirs, files in os.walk("D:\\Maimy\\CNN\\NewBatikTrain\\DataSet"):
basename = oooos.path.basename(path)
for name in files:
count += 1
if (basename != 'B1'):
print()
else:
break
print(count)
trpick = sys.argv[1]
vapick = sys.argv[2]
class_amount = sys.argv[3]
montime = sys.argv[4]
batchtime = sys.argv[5]
print("montime:", montime)
batikset = set()
for x in range(1, int(class_amount) + 1):
batikset.add("B" + str(x))