def prepareVideo(rpath, wpath):
videos = walk(rpath, targetExtensions=Extensions.videos()).get("extensions")
for vset in videos:
vpath = os.path.join(rpath, *vset)
for frame in generateFrames(vpath):
cv2.imwrite(os.path.join(wpath, "negative-{}{}".format(uuid.uuid1(), Extensions.jpg)), frame)
def frameFolderSmart(folderPath, ctgLimit):
processedVideos = readLines(Path.processedFiles)
fullCategories = readLines(Path.fullCategories)
videos = [
video for video in os.listdir(folderPath)
if video and video.endswith(Extensions.videos())
]
videosByCtgs = getSameCtgVideo(fullCategories, videos)
pass
def summarizeInfo(rawPath=Path.raw, summarizedPath=Path.summarizedRaw, allowedCategories=None, allowedSubCtgList=None,
overwrite=True):
summarized = openJsonSafely(summarizedPath) if not overwrite else {}
rawVideosPath = os.path.join(rawPath, const.videos)
rawJsonsPath = os.path.join(rawPath, const.json)
rawVideos = sorted([j for j in os.listdir(rawVideosPath) if j.endswith(Extensions.videos())])
maxIdx = summarized.get(const.maxIdx, 0)
for i, video in enumerate(rawVideos):
print(f"\rProcessing {video} ({i + 1} out of {len(rawVideos)})", end="")
category, name = extractCategory(video)
if category not in allowedCategories:
continue
categoryInfo = summarized.get(category, {})
videoJson = os.path.join(rawJsonsPath, makeJSONname(name))
videoMarks = getVideoMarks(os.path.join(rawVideosPath, video), videoJson)
for subctg, subctgMarks in videoMarks.items():
if allowedSubCtgList is not None and subctg not in allowedSubCtgList:
continue
if subctg not in categoryInfo:
subctgIdx = maxIdx
maxIdx += 1
curSubctgMarks = {
const.overall: 0,
const.ctgIdx: subctgIdx,
const.videos: {},
const.parent: category
}
else:
curSubctgMarks = categoryInfo[subctg]
if video not in curSubctgMarks[const.videos]:
curSubctgMarks[const.videos][video] = subctgMarks
curSubctgMarks[const.overall] += len(subctgMarks)
categoryInfo[subctg] = curSubctgMarks
if categoryInfo:
summarized[category] = categoryInfo
summarized[const.maxIdx] = maxIdx
json.dump(summarized, open(summarizedPath, "w"), indent=3)
print(f"\n{Fore.GREEN}Summarized info file {summarizedPath} has been updated{Style.RESET_ALL}")
def generateFramesAndTxtFromVideos(srcPath, dstPath, typeAnnotation, groups):
for group in os.listdir(srcPath):
if not group in groups:
print(f"The group {group} was skip in path {srcPath, group}\n")
continue
targetDir = os.path.join(dstPath, group)
os.makedirs(targetDir, exist_ok=True)
for video in os.listdir(os.path.join(srcPath, group)):
print(f"\n Start process file: {group}/{video} \n")
videoName, ext = os.path.splitext(video)
if not ext in list(Extensions.videos()):
continue
category = ""
videoPath = os.path.join(srcPath, group, video)
if typeAnnotation == Annotation.json:
if group == "original":
with open(os.path.join(srcPath, group, "annotation", f"{videoName}.json"), "r") as f:
jData[video] = json.load(f)
category, _ = videoName.rsplit('-', 1)
if typeAnnotation == Annotation.txt:
if group == "mix" or group == "sets":
category = videoName
if group == "original":
t = videoName.split("-")
category = t[0]
globalFrameIdx = 0
if category in catIdx:
globalFrameIdx = catIdx[category]["globalFrameIdx"]
if category == "":
print(f"Unknowed group {category} or type.Annotation {typeAnnotation} \n")
continue
framesDir = os.path.join(targetDir, category, 'frames')
os.makedirs(framesDir, exist_ok=True)
continue
frameIdx = framingVideo(typeAnnotation, videoPath, framesDir,
frame_idx=globalFrameIdx)
if group == "original":
catIdx[category]["globalFrameIdx"] = frameIdx
def actualizeInfoWithFrames(datasetPath):
print("\nActualizing info...")
actualInfo = {}
os.makedirs(os.path.dirname(Path.actualInfo), exist_ok=True)
frames = walk(datasetPath, targetDirs=const.frames)
frames = frames.get("dirs")
for idx, dirsList in enumerate(frames):
dirsList = dirsList[:-1]
fullpath = os.path.join(datasetPath, *dirsList)
images = walk(fullpath, targetExtensions=Extensions.images()).get("extensions") # TODO : некоторые картинки могут не содержать категории
putNested(dictionary=actualInfo, keys=dirsList, value=len(images))
dirsList[-1] = const.overall
updateNested(dictionary=actualInfo, keys=dirsList, value=len(images))
print("\r{:.1f}% of work has been done".format((idx + 1) / len(frames) * 100), end="")
print()
json.dump(actualInfo, open(Path.actualInfo, "w"), indent=3)
def prepareImages(rpath, wpath):
images = walk(rpath, targetExtensions=Extensions.images()).get("extensions")
for iset in images:
ipath = os.path.join(rpath, *iset)
os.rename(ipath, os.path.join(wpath, "negative-{}{}".format(uuid.uuid1(), Extensions.jpg)))
def makeSets(directories,
wpath=Path.sets,
trainPart=0.9,
validPart=0.05,
ignoreOld=False,
matchWithMarks=True):
assert 0 < trainPart + validPart <= 1
os.makedirs(wpath, exist_ok=True)
testPart = 1 - trainPart - validPart
sets = {
const.train: {
"path": os.path.join(wpath, extendName(const.train,
Extensions.txt)),
"part": trainPart,
"content": []
},
const.valid: {
"path": os.path.join(wpath, extendName(const.valid,
Extensions.txt)),
"part": validPart,
"content": []
},
const.test: {
"path": os.path.join(wpath, extendName(const.test,
Extensions.txt)),
"part": testPart,
"content": []
}
}
inUse = []
for set_, info in sets.items():
info["content"] = readLines(info["path"]) if not ignoreOld else []
inUse.extend(info["content"])
images = []
marks = []
for dirIdx, path in enumerate(directories):
print(
"\rSearching for images and marks in listed directories, {:.1f}% has been done"
.format(dirIdx / len(directories) * 100),
end="")
dirImages = [
os.path.join(path, *img) for img in walk(
path, targetExtensions=Extensions.images()).get("extensions")
]
images.extend(dirImages)
if matchWithMarks:
dirMarks = [
os.path.join(path, *mrk) for mrk in walk(
path, targetExtensions=Extensions.txt).get("extensions")
]
marks.extend(dirMarks)
if matchWithMarks:
transformer = lambda x: changeExtension(x, Extensions.txt)
print("Matching images to marks, please wait...")
images = matchLists(master=marks,
slave=images,
transformer=transformer)
# _, images = matchLists(master=inUse, slave=images, getMismatched=True)
images = permutate(images)
start = 0
for set_, info in sets.items():
part = info["part"]
end = start + int(part * len(images))
total = end - start
info["content"].extend(images[start:end])
info["content"] = permutate(info["content"])
start = end
writeLines(lines=info["content"], path=info["path"])
print(f"\n{Fore.GREEN}Added {total} paths to {set_} {Style.RESET_ALL}")