# encoding=utf8
import os
import shutil
from basicFun import FILES
from tqdm import tqdm
if __name__ == "__main__":
count=0
srcDir=r"/DATACENTER4/hao.yang/project/Qin/data/imgs/safe/safe_CFMXX_img_divided/"
tarDir=r"/DATACENTER4/hao.yang/project/Qin/data/imgs/safe/safe_FMXX_img/"
allJpgs=[x for x in FILES.list_all_filePaths(srcDir) if '.jpg' in x]
FILES.mkdir(tarDir)
for jpgPath in tqdm(allJpgs):
shutil.copy(jpgPath,os.path.join(tarDir,jpgPath.split('/')[-1]))
count+=1
print("copy jpgs:",count)
sites = ['FengShi', 'MingDeMen', 'XiWan']
for site in sites:
for isle in range(1, 6):
outXmlDir = "/disk2/hao.yang/project/Qin/data/imgs/isle/{}/isle{}_{}_img/".format(
site, isle, site)
volume = 1000
partition = 1
allFiles = FILES.get_sorted_files(outXmlDir)
random.shuffle(allFiles)
count = 0
for file in allFiles:
if count < volume + volume * (partition - 1) * 0.2:
count += 1
else:
# exit()
# break
partition += 1
count = 0
# tar_outXmlDir='{}_partition{}'.format(outXmlDir,partition)
pDir = '/disk2/hao.yang/project/Qin/data/labelTask/isles/p{}'.format(
partition)
FILES.mkdir(pDir)
tar_outXmlDir = '/disk2/hao.yang/project/Qin/data/labelTask/isles/p{}/{}_isle{}'.format(
partition, site, isle)
FILES.mkdir(tar_outXmlDir)
filePath = os.path.join(outXmlDir, file)
tarPath = os.path.join(tar_outXmlDir, file)
shutil.copy(filePath, tarPath)
# copy_files_refer_dir(jpgDir,'.jpg',outXmlDir,jpgTar)
bbox.append(bndbox['xmin'])
#y
bbox.append(bndbox['ymin'])
#w
bbox.append(bndbox['xmax'] - bndbox['xmin'])
#h
bbox.append(bndbox['ymax'] - bndbox['ymin'])
addAnnoItem(object_name, current_image_id,
current_category_id, bbox)
if __name__ == '__main__':
xml_path = '/media/kevin/办公/xizang/testdataset/1028/val_/'
jsonDir = './{}/'.format('val')
print(xml_path)
FILES.mkdir(jsonDir)
if jsonDir:
if xml_path[-1] == '/':
xmlName = xml_path.split('/')[-2]
else:
xmlName = xml_path.split('/')[-1]
json_file = os.path.join(jsonDir, xmlName + '.json')
else:
if xml_path[-1] == '/':
json_file = xml_path[:-1] + '.json'
else:
json_file = xml_path + '.json'
setCatItem(labelmap)
parseXmlFiles(xml_path)
json.dump(coco, open(json_file, 'w'))
for file in allFiles:
if count - remRate > 0:
filePath = os.path.join(sampleDir, file)
tarPath = os.path.join(tarDir, file)
shutil.copy(filePath, tarPath)
fileNum += 1
remRate += REMRATE
count += 1
memberRoot = '/disk2/hao.yang/project/Qin/data/xmls/guide'
members = FILES.get_sub_dirs(memberRoot)
for member in members:
sampleRoot = os.path.join(memberRoot, member)
tarRoot = os.path.join(memberRoot, member + '_sample')
FILES.mkdir(tarRoot)
dirNames = FILES.get_sub_dirs(sampleRoot)
for dirName in dirNames:
sampleDir = os.path.join(sampleRoot, dirName)
tarDir = os.path.join(tarRoot, dirName)
FILES.mkdir(tarDir)
sample_files(sampleDir, tarDir, 2000)
# xmlDir=r"E:\factory\voc\add_data\summary\rawImg\Tank_jl"
# jpgDir=r"E:\research\lackLabel\voc\img4206"
# xmlTar=r"E:\factory\voc\add_data\summary\rawImg\Tank_jl_lite"
# jpgTar=r"E:\research\lackLabel\voc\img2103_1"
# remainNum=70
# FILES.rm_mkdir(xmlTar)
# # FILES.rm_mkdir(jpgTar)
# allFiles=FILES.get_sorted_files(xmlDir)
# filesCount=len(allFiles)
#二级目录下的jpg文件复制到一个文件夹
import os
import shutil
import time
from basicFun import FILES
if __name__=="__main__":
timeStart=time.clock()
roots="/DATACENTER4/hao.yang/project/Qin/data/preProcess/extractFrame/"
rootsName=['0706_checkout']
for rootName in rootsName:
rootdir=os.path.join(roots,rootName)
tardir=os.path.join(roots,rootName+"All") #不用创建
print(rootdir,tardir)
i=1
if os.path.exists(rootdir):
FILES.mkdir(tardir)
alldirs=FILES.get_sub_dirs(rootdir)
for dirName in alldirs:
if 'test' in dirName:
continue
dirpath=os.path.join(rootdir,dirName)
# print(dirpath)
alljpg=sorted([x for x in FILES.get_files(dirpath) if ".jpg" in x])#审核为.jpg形成列表
for jpg in alljpg:
# rename
# newJpg=dirpath.split("/")[-1]+str("%06d"%i)+".jpg"
newJpg='{}_{}'.format(dirName,jpg)
# newJpg=jpg
i+=1
oldpath=os.path.join(dirpath,jpg)
newpath=os.path.join(tardir,newJpg)
import os, shutil
from basicFun import FILES
dir1 = r"/disk2/hao.yang/project/Qin/data/preProcess/tolabelImg/0501/safe_cashbox_FengShi_simulate/"
des1 = r"/disk2/hao.yang/project/Qin/data/preProcess/tolabelImg/0501/safe_cashbox_FengShi_simulate_img/"
dir2 = '/disk2/hao.yang/project/Qin/data/preProcess/tolabelImg/0501/xml/'
des2 = '/disk2/hao.yang/project/Qin/data/preProcess/tolabelImg/0501/labeled_tube_px/'
FILES.mkdir(des1)
FILES.mkdir(des2)
preTitle = 'safe_cashbox_FengShi_simulate_'
endTitle = ''
i = 0
filelist = FILES.get_sorted_files(dir1)
for file in filelist:
if ".jpg" in file:
newfileName = '%06d' % i
# newfileName=file
# shutil.copy(os.path.join(dir1,file),os.path.join(des1,preTitle+newfileName+endTitle+".jpg"))
shutil.copy(os.path.join(dir1, file),
os.path.join(des1, preTitle + file))
if os.path.exists(
os.path.join(
dir2, 'unload_FengShi_tube_simulate_' + newfileName +
endTitle + ".xml")):
shutil.copy(
os.path.join(
dir2, 'unload_FengShi_tube_simulate_' + newfileName +
endTitle + ".xml"),
os.path.join(des2, preTitle + file.replace('.jpg', ".xml")))
else:
print(file, "no xml")
i += 1
print("cate: {} MinW={}, MinH={}, MinA={}".format(cate, MinW, MinH, MinA))
def find_xmls_with_minimal_size_of_each_cate(xmlDir):
allCates = find_all_cates(xmlDir)
for cate in allCates:
find_xmls_with_minimal_size_of_this_cate(xmlDir, cate)
if __name__ == "__main__":
xmlDir = r'/DATACENTER4/hao.yang/project/Qin/data/xmls/checkout/xml_checkout_10972_total/'
tarDir = '/DATACENTER4/hao.yang/project/Qin/data/xmls/checkout/pos_use/'
FILES.rm_mkdir(tarDir)
# Uncomment to find some xmls containing all categories and copy matching jpgs
# find_xmls_with_all_cates(xmlDir)
# Uncomment to find xmls and jpgs containing given cates
cates = ['pos_use']
# filter_short_cates(xmlDir,cates)
filter_xmls_with_cates(xmlDir, cates)
# move_xmls_with_cates(xmlDir,cates)
# filter_xmls_without_cates(xmlDir,cates)
# Uncomment to find boxes with minimal area, minimal width and minimal height of each category
# find_xmls_with_minimal_size_of_each_cate(xmlDir)
# Uncomment to filter xml and jpg whose amount of label_person > label_helmet
# filter_lack_helmet(xmlDir)
# Uncomment to copy jpgs referring xmls
jpgDir = r"/DATACENTER4/hao.yang/project/Qin/data/imgs/checkout/imgs_labeled_all_10972/"
jpgTar = tarDir
FILES.mkdir(jpgTar)
FILES.shutil_by_refer(tarDir, '.xml', '.jpg', jpgDir, jpgTar)
# else:
# dstRoot=jpgRoot+"Unique"+str(int(SIMIBOUND*1000))
# dstRoot=jpgRoot.replace('0505','0505removedSimilarityFrame')
dstRoot = jpgRoot.replace('extractFrame', 'removedSimilarityFrame')
rootsName = FILES.get_sub_dirs(jpgRoot)
# rootsName=[1]
startdir = 1 #从第startdir开始，处理完第enddir结束（包含enddir）
enddir = 43 #注意作为开始的startdir第一帧无事件
# 需定义的参数：
if __name__ == "__main__":
# IMG.info()
start = time.time()
count = 1
resume = 1
dirCount = 1
FILES.mkdir(dstRoot)
for rootName in rootsName:
dirpath = os.path.join(jpgRoot, rootName)
# dirpath="/DATACENTER5/hao.yang/dataRoom/Qin/park/caotan/longVideoFrames/8/"
if os.path.exists(dirpath):
difdirpath = dirpath.replace('extractFrame',
'removedSimilarityFrame')
# if difdirpath[-1]=='/':
# difdirpath=difdirpath[:-1]+"Unique"+str(int(SIMIBOUND*1000))
# else:
# difdirpath=difdirpath+"Unique"+str(int(SIMIBOUND*1000))
if dirCount < startdir:
dirCount += 1
continue
if dirCount > enddir:
break
members = ['ph', 'xgt', 'yxy', 'zt', 'hyd', 'hjc']
membersLimits = {
'ph': 267,
'xgt': 318,
'yxy': 360,
'zt': 259,
'hyd': 276,
'hjc': 413
}
taskImg = r'E:\factory\voc\helmet_img'
taskXml = r'E:\factory\voc\labeled_xml_lackHel\xml'
outRoot = r'E:\factory\voc\labeled_xml_lackHel_assignTask'
for member in members:
outDir = outRoot + '/' + member
FILES.rm_mkdir(outDir)
FILES.mkdir(outDir + '/img')
FILES.mkdir(outDir + '/xml')
allXmls = FILES.get_sorted_files(taskXml)
random.shuffle(allXmls)
memberI = 0
memberF = 0
for xml in allXmls:
print(memberI)
member = members[memberI]
desDir = outRoot + '/' + member
xmlPath = taskXml + '/' + xml
desXml = desDir + '/xml/' + xml
jpg = xml.split('.')[0] + '.jpg'
jpgPath = taskImg + '/' + jpg
desJpg = desDir + '/img/' + jpg
# copy file
if __name__ == "__main__":
start = time.time()
count = 1
resume = 1
dirpath = '/DATACENTER5/hao.yang/dataRoom/Qin/safe/caotan/data_auto/stillImg'
if os.path.exists(dirpath):
difdirpath = dirpath + '_rmsimilar' + str(int(SIMIBOUND * 100)) #不用创建
# for dir in alldirs:
# if dircount<startdir:
# dircount += 1
# continue
# if dircount>enddir:
# break
begin = 0
FILES.mkdir(difdirpath)
print(dirpath, "copyRemoving...")
# allfiles=getAllfiles(dirpath)
allfiles = sorted(os.listdir(dirpath))
for file in allfiles:
if ".jpg" in file:
filepath = os.path.join(dirpath, file)
print(filepath)
if begin == 0:
difpath = os.path.join(difdirpath, file)
shutil.copyfile(filepath, difpath)
basepath = filepath
begin = 1
continue
if count < resume:
count += 1
#encoding=utf-8
'assign img task to members as sequence'
import os
import shutil
import random
# personal lib
from basicFun import FILES
if __name__ == "__main__":
namei = 0
members = ['day_1', 'day_2', 'day_3', 'day_4']
taskDir = '/DATACENTER2/ke.cao/oil_video_Data/unload_images_dif'
outRoot = taskDir + '_day'
FILES.mkdir(outRoot)
allImgs = FILES.get_sorted_files(taskDir)
# 每人大概分配的数量
perAmount = int(len(allImgs) / len(members))
# 可能会有分不匀的情况
redundant = len(allImgs) % len(members)
print('{}*{}+{}=?{}'.format(perAmount, len(members), redundant,
len(allImgs)))
assignCount = 0
for member in members:
outDir = os.path.join(outRoot, member)
FILES.mkdir(outDir)
memberAmount = perAmount
for img in allImgs:
if assignCount < memberAmount:
# print(members[namei])
desDir = os.path.join(outRoot, members[namei])
# jpg move
srcPath = os.path.join(taskDir, img)
if resume > 0:
resume -= 1
continue
if begin == 0:
baseName = checkName
begin = 1
continue
if checkTitle != baseTitle:
distance = 1000
baseTitle = checkTitle
else:
distance = 100
try:
distance = calculate_similarity_on_labels(baseName, checkName)
except:
FILES.mkdir(errorDir)
checkImg = os.path.join(imgDir,
checkName.replace('.xml', '.jpg'))
errorImg = os.path.join(errorDir,
checkName.replace('.xml', '.jpg'))
checkXml = os.path.join(xmlDir, checkName)
errorXml = os.path.join(errorDir, checkName)
shutil.copy(checkImg, errorImg)
shutil.copy(checkXml, errorXml)
baseImg = os.path.join(imgDir,
baseName.replace('.xml', '.jpg'))
errorImg = os.path.join(errorDir,
baseName.replace('.xml', '.jpg'))
baseXml = os.path.join(xmlDir, baseName)
errorXml = os.path.join(errorDir, baseName)
shutil.copy(baseImg, errorImg)
if objName == cateName:
filteredXmls.append(xml)
viewCount += 1
sys.stdout.write('\rFiltering xmls >>{count:.2f}%'.format(
count=100 * float(viewCount) / xmlSum))
# sys.stdout.write('\r>> Detecting {name} {count:.2f}%'.format(name=videoPath.split('/')[-1],count=(100*float(i)/total_frames)))
sys.stdout.flush()
return filteredXmls
xmlDir = r"/DATACENTER2/hao.yang/project/Qin/checkout/FengShi/xml1383_8kinds/"
if xmlDir[-1] == '/':
xmlTar = xmlDir[:-1] + '_val'
else:
xmlTar = xmlDir + '_val'
FILES.mkdir(xmlTar)
# Get leastNum boxes of the cate at least
leastNum = 40
count = 0
valCount = 0
allXmls = FILES.get_sorted_files(xmlDir)
# Filter xmls having a box of a cate
cateName = 'pos_idle'
print('Start to filter xmls containing cate {}'.format(cateName))
filteredXmls = filter_by_cate(allXmls, cateName)
print('\nTotally {} xmls containing cate {}'.format(len(filteredXmls),
cateName))
valRate = int(len(filteredXmls) / leastNum)
for xml in filteredXmls:
if count % valRate == 0:
xmlPath = os.path.join(xmlDir, xml)
#encoding=utf-8
'assign img & xml task to members'
import os
import shutil
import random
# personal lib
from basicFun import FILES
if __name__ == "__main__":
namei = 0
evenMembers = ['szl', 'hs']
limitMembers = {'zzh': 60}
taskDir = r'/DATACENTER4/hao.yang/project/Qin/data/preProcess/removedSimilarityFrame/0706/checkout/labeling/0706_checkout_img_hand_scanner/'
outRoot = r'/DATACENTER4/hao.yang/project/Qin/data/preProcess/removedSimilarityFrame/0706/checkout/labeling/0706_checkout_assign_hand_scanner/'
for member in limitMembers:
outDir = os.path.join(outRoot, member)
FILES.mkdir(outDir)
for member in evenMembers:
outDir = os.path.join(outRoot, member)
FILES.mkdir(outDir)
allImgs = FILES.get_sorted_files(taskDir)
totalImg = len(allImgs)
assignIndex = 0
memberI = 0
memberF = 0
# Assign to Limit Members
for img in allImgs:
# print(memberI)
member = list(limitMembers.keys())[memberI]
desDir = os.path.join(outRoot, member)
jpgPath = os.path.join(taskDir, img)
desJpg = os.path.join(desDir, img)
# 可能会有分不匀的情况
redundant = len(allImgs) % len(members)
print('{}*{}+{}=?{}'.format(perAmount, len(members), redundant,
len(allImgs)))
assignCount = 0
for member in members:
outDir = os.path.join(outRoot, '{}_{}'.format(preTitle, member))
FILES.rm_mkdir(outDir)
memberAmount = perAmount
for img in allImgs:
if assignCount < memberAmount:
# print(members[namei])
desDir = os.path.join(
outRoot, '{}_{}/{}_{}_img'.format(preTitle, members[namei],
preTitle, members[namei]))
FILES.mkdir(desDir)
srcPath = os.path.join(jpgDir, img)
desPath = os.path.join(desDir, img)
shutil.copy(srcPath, desPath)
desDir = os.path.join(
outRoot, '{}_{}/{}_{}_xml'.format(preTitle, members[namei],
preTitle, members[namei]))
FILES.mkdir(desDir)
srcPath = os.path.join(xmlDir, img.replace('.jpg', '.xml'))
if os.path.exists(srcPath):
desPath = os.path.join(desDir, img.replace('.jpg', '.xml'))
shutil.copy(srcPath, desPath)
assignCount += 1
else:
print('member:{} amount:{}'.format(members[namei], assignCount))
assignCount = 0