def extract_features_from_video_annotation_pair(vap):
d = T.get_annotated_frames_w_numbers(vap, "cup")
frame_numbers = list(d['frame-numbers'])
bboxs = np.asarray(d['filtered-anno'])[:, :4]
ve = VE.videoExplorer()
ve.setVideoStream(vap["video_file"], frameMode='RGB')
feats = []
lbls = []
# counter for bbox index
cnt = 0
for i, frame in enumerate(ve):
if i not in frame_numbers:
continue
if T.contains_nan(bboxs[cnt]):
cnt += 1
continue
fld = FC.extract_features_for_training(frame, bboxs[cnt])
cnt += 1
if fld is None:
# cup present, but could not get segmented reliably
continue
feats += [fld['features']]
lbls += [fld['labels']]
return {"features": np.concatenate(feats),
"labels": np.concatenate(lbls)}
def retrieveVideoLength(self, filename, initialStepSize=10000):
"""
Finds video length by accessing it bruteforce
"""
idx = 0
modi = initialStepSize
vE = VE.videoExplorer()
if self.vialNames == [None]:
filename = filename.split('.bhvr')[0] + self.videoExtension
else:
filename = filename.split('.bhvr')[0] + '.v{0}{ext}'.format(
self.vialNames[0],
ext=self.videoExtension)
while modi > 0:
while True:
try:
vE.getFrame(filename, frameNo=idx, frameMode='RGB')
except StopIteration:
break
idx += modi
idx -= modi
modi /= 2
return idx + 1 # + 1 to make it the same behaviour as len()
def __init__(self, videoFolder, backgroundFolder, patchFolder, flyClassifierPath, noveltyClassfyPath,
recIdx, runIdx, minPerRun=20, ffmpegpath=None):
self.flyClassifierPath = flyClassifierPath
self.noveltyClassfyPath = noveltyClassfyPath
self.setupFlyClassifiers()
self.vE = VE.videoExplorer()
self.bgModel = BG.backgroundModel(verbose=False, colorMode='rgb')
self.viewer = IV.imgViewer()
self.roi = [[350, 660], [661, 960], [971, 1260], [1270, 1600]]
self.vial = V.Vials(self.roi, gaussWeight=2000, sigma=20,
xoffsetFact=0.6, clfyFunc=self.flyClassify,
acceptPosFunc=V.Vials.acceptPosFunc,
acceptPosFuncArgs=self.acceptArgs, ffmpegpath=ffmpegpath)
self.bgModel = BG.backgroundModel(verbose=False, colorMode='rgb')
self.videoFolder = videoFolder
self.backgroundFolder = backgroundFolder
self.patchFolder = patchFolder
self.recIdx = recIdx
self.recIdx = recIdx
self.minPerRun = minPerRun
self.runIdx = runIdx
self.bgSampleSize = 10
def retrieveVideoLength(self, filename, initialStepSize=10000):
"""
Finds video length by accessing it bruteforce
"""
idx = 0
modi = initialStepSize
vE = VE.videoExplorer()
if self.vialNames == [None]:
filename = filename.split('.bhvr')[0] + self.videoExtension
else:
filename = filename.split('.bhvr')[0] + '.v{0}{ext}'.format(
self.vialNames[0], ext=self.videoExtension)
while modi > 0:
while True:
try:
vE.getFrame(filename, frameNo=idx, frameMode='RGB')
except StopIteration:
break
idx += modi
idx -= modi
modi /= 2
return idx + 1 # + 1 to make it the same behaviour as len()
def plot_random_cup_predictions(vap, classifier):
d = T.get_annotated_frames_w_numbers(vap)
frame_numbers = list(d['frame-numbers'])
bboxs = np.asarray(d['filtered-anno'])[:, :4]
selected_frames = np.random.permutation(np.arange(len(frame_numbers)))[:10]
ve = VE.videoExplorer()
ve.setVideoStream(vap["video_file"], frameMode='RGB')
feats = []
lbls = []
for i in selected_frames:
frame = ve.getFrame(vap["video_file"], frame_numbers[i], frameMode='RGB')
fsd = FC.extract_features(frame) # fsd: feature_segment_dict
feats = fsd['features']
lbls = classifier.predict_proba(feats)
print np.sum(lbls)
segments = fsd['segments']
plot_cup_prediction(frame, segments, lbls)
plt.title("frame {} of {}".format(i, vap["video_file"]))
def cup_predictions(vap, classifier):
d = T.get_annotated_frames_w_numbers(vap, "cup")
frame_numbers = list(d['frame-numbers'])
bboxs = np.asarray(d['filtered-anno'])[:, :4]
selected_frames = frame_numbers
ve = VE.videoExplorer()
ve.setVideoStream(vap["video_file"], frameMode='RGB')
data = []
for i, frame_no in enumerate(selected_frames):
if i == 0 or (selected_frames[i - 1] + 1) != selected_frames[i]:
frame = ve.getFrame(vap["video_file"], frame_no, frameMode='RGB')
else:
frame = ve.next()
ccp = predict_cup_center(frame, classifier)
confidences = ccp["confidences"]
centroids = ccp["centroids"]
cc = ccp["winner"]
data += [[frame_no, cc[1], cc[0]]]
df = pd.DataFrame(data=data, columns=["Frame", "cup x", "cup y"])
return df
def getVideoPaths(self, rootPath, start, end, sampleSize=200):
"""
Parse video files in the root folder
Args:
rootPath (string):
path to root folder
startDate (datetime):
beginning date
endDate (datetime):
end date
sampleSize (int):
number of frames used to estimate background
"""
self.rootPath = rootPath
self.startDate = start
self.endDate = end
self.vE = videoExplorer(verbose=self.verbose, rootPath=rootPath)
self.vE.setTimeRange(self.startDate, self.endDate)
self.vE.parseFiles()
self.dayPaths = self.vE.getPathsOfList(self.vE.dayList)
self.nightPaths = self.vE.getPathsOfList(self.vE.nightList)
def getVideoPaths(self, rootPath, start, end, sampleSize=200):
"""
Parse video files in the root folder
Args:
rootPath (string):
path to root folder
startDate (datetime):
beginning date
endDate (datetime):
end date
sampleSize (int):
number of frames used to estimate background
"""
self.rootPath = rootPath
self.startDate = start
self.endDate = end
self.vE = videoExplorer(verbose=self.verbose, rootPath=rootPath)
self.vE.setTimeRange(self.startDate, self.endDate)
self.vE.parseFiles()
self.dayPaths = self.vE.getPathsOfList(self.vE.dayList)
self.nightPaths = self.vE.getPathsOfList(self.vE.nightList)
def findOverlongVideos(root_folder,
pos_folder,
anno_folder,
video_extension='mp4'):
import pyTools.system.videoExplorer as VE
import numpy as np
import yaml
vE = VE.videoExplorer()
anno = Annotation()
unequalFiles = {}
for root, dirs, files in sorted(os.walk(root_folder)):
for filename in sorted(files):
if filename.endswith(video_extension):
anno_filename = filename[:-len(video_extension)] + 'csv'
pos_filename = filename[:-len(video_extension)] + 'pos.npy'
af = os.path.join(os.path.dirname(root), anno_folder,
anno_filename)
pf = os.path.join(os.path.dirname(root), pos_folder,
pos_filename)
videoLength = vE.retrieveVideoLength(
os.path.join(root, filename))
try:
posLength = len(np.load(pf))
except IOError:
print "No posfile for {}".format(pf)
# continue
if videoLength > 1800:
if posLength != (videoLength):
print "pos f**k:\nfile: {}\nvL: {}\npL: {}".format(
filename, videoLength, posLength)
try:
anno.loadFromFile(af)
except IOError:
continue
annoLength = anno.getLength()
if annoLength != videoLength:
print "anno f**k:\nfile: {}\nvL: {}".format(
filename, videoLength)
unequalFiles[filename] = [videoLength
] #, posLength, annoLength]
yamlStr = yaml.dump(unequalFiles,
default_flow_style=False,
encoding=('utf-8'))
with open('/Users/peter/Desktop/files.json', 'w') as f:
f.writelines(yamlStr)
def alignVideoLengthToRest(root_folder,
pos_folder,
anno_folder,
video_extension='mp4'):
import pyTools.system.videoExplorer as VE
import numpy as np
import yaml
vE = VE.videoExplorer()
anno = Annotation()
for root, dirs, files in sorted(os.walk(root_folder)):
for filename in sorted(files):
if filename.endswith(video_extension):
anno_filename = filename[:-len(video_extension)] + 'csv'
pos_filename = filename[:-len(video_extension)] + 'pos.npy'
af = os.path.join(os.path.dirname(root), anno_folder,
anno_filename)
pf = os.path.join(os.path.dirname(root), pos_folder,
pos_filename)
videoLength = vE.retrieveVideoLength(
os.path.join(root, filename))
try:
pos = np.load(pf)
posLength = len(pos)
if posLength != (videoLength):
newPos = pos[:videoLength, :]
np.save(pf, newPos)
except IOError:
print "No posfile for {}".format(pf)
continue
try:
anno.loadFromFile(af)
annoLength = anno.getLength()
if annoLength != videoLength:
print "anno f**k:\nfile: {}\nvL: {}\naL: {}".format(
filename, videoLength, annoLength)
anno.removeAnnotation(
vial=None,
frames=annoLength,
annotator='automatic placeholder',
behaviour='video length')
anno.addAnnotation(vial=None,
frames=[videoLength],
annotator='automatic placeholder',
behaviour='video length')
anno.saveToFile(af)
except IOError:
print "no annotation file for: {}".format(filename)
continue
def __init__(self, posPath, idxSlice, selectedVials=[1],
thread=None, eofCallback=None):
super(VideoLoader, self).__init__(None)
self.vE = VE.videoExplorer()
self.posPath = None
self.init(posPath, idxSlice, selectedVials, thread)
def extractFeatures(self):
"""
Iteration over video file to extract features
:return:
"""
if len(self.videoListRel) == 1:
progressSteps = VE.videoExplorer.retrieveVideoLength(
self.rel2absFile(self.videoListRel[0]))
else:
progressSteps = len(self.videoListRel)
self.setStatus("extracting features..", progressSteps)
for i, relVidFile in enumerate(self.videoListRel):
if os.path.exists(self.generateColorFeatureSavePath(relVidFile))\
and os.path.exists(self.generateDiffImgFeatureSavePath(relVidFile)):
# do not recompute features already computed
if len(self.videoListRel) > 1:
self.incrementStatus()
continue
colorFeat = []
diffImgFeat = []
vE = VE.videoExplorer()
vE.setVideoStream(self.rel2absFile(relVidFile), frameMode='RGB')
anno = A.Annotation()
anno.loadFromFile(
self.rel2absFile(self.convertVideoToAnnoFilename(relVidFile)))
annotationList = anno.filterFrameLists(self.annotationFilters)
prevFrame = None
for frame in vE:
colorFeat += [
self.extractColorFeatureFromFrame(frame, annotationList[i])
]
if prevFrame:
diffImgFeat += [
self.extractDiffImgFeatureFromFrames(
prevFrame, frame, annotationList[i])
]
else:
diffImgFeat += [np.zeros((16, ), dtype=np.int32)]
if len(self.videoListRel) == 1:
self.incrementStatus()
np.save(self.generateColorFeatureSavePath(relVidFile),
np.asarray(colorFeat))
np.save(self.generateDiffImgFeatureSavePath(relVidFile),
np.asarray(diffImgFeat))
if len(self.videoListRel) > 1:
self.incrementStatus()
def extractHOGfromVideo(path):
vE = videoExplorer()
vE.setVideoStream(path, info=False, frameMode='RGB')
feats = []
for frame in vE:
feats += [feat.hog(frame)]
f = np.asarray(feats)
np.save(path.split('.avi')[0] + '.feat.{ending}.npy'.format(ending='hog-8'), f)
def extractFeatures(self):
"""
Iteration over video file to extract features
:return:
"""
if len(self.videoListRel) == 1:
progressSteps = VE.videoExplorer.retrieveVideoLength(
self.rel2absFile(self.videoListRel[0]))
else:
progressSteps = len(self.videoListRel)
self.setStatus("extracting features..", progressSteps)
for i, relVidFile in enumerate(self.videoListRel):
if os.path.exists(self.generateColorFeatureSavePath(relVidFile))\
and os.path.exists(self.generateDiffImgFeatureSavePath(relVidFile)):
# do not recompute features already computed
if len(self.videoListRel) > 1:
self.incrementStatus()
continue
colorFeat = []
diffImgFeat = []
vE = VE.videoExplorer()
vE.setVideoStream(self.rel2absFile(relVidFile), frameMode='RGB')
anno = A.Annotation()
anno.loadFromFile(self.rel2absFile(
self.convertVideoToAnnoFilename(relVidFile)))
annotationList = anno.filterFrameLists(self.annotationFilters)
prevFrame = None
for frame in vE:
colorFeat += [self.extractColorFeatureFromFrame(frame,
annotationList[i])]
if prevFrame:
diffImgFeat += [self.extractDiffImgFeatureFromFrames(prevFrame,
frame,
annotationList[i])]
else:
diffImgFeat += [np.zeros((16,), dtype=np.int32)]
if len(self.videoListRel) == 1:
self.incrementStatus()
np.save(self.generateColorFeatureSavePath(relVidFile),
np.asarray(colorFeat))
np.save(self.generateDiffImgFeatureSavePath(relVidFile),
np.asarray(diffImgFeat))
if len(self.videoListRel) > 1:
self.incrementStatus()
def __init__(self,
posPath,
idxSlice,
selectedVials=[1],
thread=None,
eofCallback=None):
super(VideoLoader, self).__init__(None)
self.vE = VE.videoExplorer()
self.posPath = None
self.init(posPath, idxSlice, selectedVials, thread)
def __init__(self, filename, bufferWidth, resultSlot):
super(VideoLengthQuery, self).__init__(None)
self.filename = filename
self.bufferWidth = bufferWidth
self.length = None
self.vE = VE.videoExplorer()
self.thread = MyThread("VideoLengthQuery")
self.moveToThread(self.thread)
self.thread.start()
self.startProcess.connect(self.retrieveVideoLength)
self.videoLength.connect(resultSlot)
self.startProcess.emit()
def __init__(self, filename, bufferWidth, resultSlot):
super(VideoLengthQuery, self).__init__(None)
self.filename = filename
self.bufferWidth = bufferWidth
self.length = None
self.vE = VE.videoExplorer()
self.thread = MyThread("VideoLengthQuery")
self.moveToThread(self.thread)
self.thread.start()
self.startProcess.connect(self.retrieveVideoLength)
self.videoLength.connect(resultSlot)
self.startProcess.emit()
def plot_cup_selection(vap):
d = T.get_annotated_frames_w_numbers(vap)
frame_numbers = list(d['frame-numbers'])
bboxs = np.asarray(d['filtered-anno'])[:, :4]
selected_frames = np.random.permutation(np.arange(len(frame_numbers)))[:10]
ve = VE.videoExplorer()
ve.setVideoStream(vap["video_file"], frameMode='RGB')
feats = []
lbls = []
for i in selected_frames:
frame = ve.getFrame(vap["video_file"], frame_numbers[i], frameMode='RGB')
plot_cup(frame, bboxs[i])
def extract_features_from_video_annotation_pair(vap):
frame_numbers = set(T.get_annotated_frames_w_numbers(vap, "card_board")["frame-numbers"])
ve = VE.videoExplorer()
ve.setVideoStream(vap["video_file"], frameMode='RGB')
feats = []
lbls = []
for i, frame in enumerate(ve):
feats += [FC.extract_histogram_features(frame)]
if i in frame_numbers:
lbls += [1]
else:
lbls += [0]
return {"features": np.asarray(feats),
"labels": np.asarray(lbls)}
def __init__(self, flyClassifier, noveltyClassfy, targetFolder=None):
self.rois = [[350, 660], [661, 960], [971, 1260], [1270, 1600]]
self.v = V.Vials(self.rois, gaussWeight=2000, sigma=20, xoffsetFact=0.6)
self.iV = IV.imgViewer()
self.vE = VE.videoExplorer()
self.update = None
self.bgImg = None
self.wasUpdated = [False for i in range(4)]
self.diffImg = None
self.frame = None
self.clfyFunc = lambda patch: np.min(patch.flatten()) < -250
self.bgModel = BM.backgroundModel(verbose=True, colorMode='rgb')
if targetFolder is None:
self.targetFolder = "/tmp/backgrounds/"
else:
self.targetFolder = targetFolder
self.noveltyClassfy = noveltyClassfy
self.flyClassifier = flyClassifier
def __init__(self,
videoFolder,
backgroundFolder,
patchFolder,
flyClassifierPath,
noveltyClassfyPath,
recIdx,
runIdx,
minPerRun=20,
ffmpegpath=None):
self.flyClassifierPath = flyClassifierPath
self.noveltyClassfyPath = noveltyClassfyPath
self.setupFlyClassifiers()
self.vE = VE.videoExplorer()
self.bgModel = BG.backgroundModel(verbose=False, colorMode='rgb')
self.viewer = IV.imgViewer()
self.roi = [[350, 660], [661, 960], [971, 1260], [1270, 1600]]
self.vial = V.Vials(self.roi,
gaussWeight=2000,
sigma=20,
xoffsetFact=0.6,
clfyFunc=self.flyClassify,
acceptPosFunc=V.Vials.acceptPosFunc,
acceptPosFuncArgs=self.acceptArgs,
ffmpegpath=ffmpegpath)
self.bgModel = BG.backgroundModel(verbose=False, colorMode='rgb')
self.videoFolder = videoFolder
self.backgroundFolder = backgroundFolder
self.patchFolder = patchFolder
self.recIdx = recIdx
self.recIdx = recIdx
self.minPerRun = minPerRun
self.runIdx = runIdx
self.bgSampleSize = 10
def findInteruptions(root_folder,
pos_folder,
anno_folder,
video_extension='mp4'):
import pyTools.system.videoExplorer as VE
import numpy as np
import yaml
vE = VE.videoExplorer()
anno = Annotation()
unequalFiles = {}
day = 1
month = 2
minute = 0
vial = 0
def incrementTime(day, month, minute, vial):
vial += 1
if vial > 3:
vial = 0
minute += 1
if minute > 59:
minute = 0
day += 1
if month == 2 and day > 28:
day = 0
month = 3
if month == 3 and day > 31:
day = 0
month = 4
return day, month, minute, vial
for root, dirs, files in sorted(os.walk(root_folder)):
for filename in sorted(files):
if not filename.endswith(video_extension):
continue
bn = os.path.basename(filename)
date = bn.split('.')[0]
cur_day = int(date.split('-')[2])
cur_month = int(date.split('-')[1])
time = bn.split('.')[1]
cur_minute = int(time.split('-')[1])
cur_vial = int(bn.split('.')[2][1])
printed = False
cnt = 0
while day != cur_day \
or month != cur_month \
or minute != cur_minute \
or vial != cur_vial:
if not printed:
printed = True
# print day, month, minute, vial
cnt += 1
day, month, minute, vial = incrementTime(
day, month, minute, vial)
if printed:
print filename, 'and next', cnt / 4, 'minutes'
day, month, minute, vial = incrementTime(day, month, minute, vial)
def alignVideoLengthToRest(root_folder, pos_folder, anno_folder,
video_extension='mp4'):
import pyTools.system.videoExplorer as VE
import numpy as np
import yaml
vE = VE.videoExplorer()
anno = Annotation()
for root, dirs, files in sorted(os.walk(root_folder)):
for filename in sorted(files):
if filename.endswith(video_extension):
anno_filename = filename[:-len(video_extension)] + 'csv'
pos_filename = filename[:-len(video_extension)] + 'pos.npy'
af = os.path.join(os.path.dirname(root),
anno_folder,
anno_filename)
pf = os.path.join(os.path.dirname(root),
pos_folder,
pos_filename)
videoLength = vE.retrieveVideoLength(os.path.join(root,
filename))
try:
pos = np.load(pf)
posLength = len(pos)
if posLength != (videoLength):
newPos = pos[:videoLength, :]
np.save(pf, newPos)
except IOError:
print "No posfile for {}".format(pf)
continue
try:
anno.loadFromFile(af)
annoLength = anno.getLength()
if annoLength != videoLength:
print "anno f**k:\nfile: {}\nvL: {}\naL: {}".format(filename,
videoLength,
annoLength)
anno.removeAnnotation(vial=None,
frames=annoLength,
annotator='automatic placeholder',
behaviour='video length')
anno.addAnnotation(vial=None,
frames=[videoLength],
annotator='automatic placeholder',
behaviour='video length')
anno.saveToFile(af)
except IOError:
print "no annotation file for: {}".format(filename)
continue
def findInteruptions(root_folder, pos_folder, anno_folder,
video_extension='mp4'):
import pyTools.system.videoExplorer as VE
import numpy as np
import yaml
vE = VE.videoExplorer()
anno = Annotation()
unequalFiles = {}
day = 1
month = 2
minute = 0
vial = 0
def incrementTime(day, month, minute, vial):
vial += 1
if vial > 3:
vial = 0
minute += 1
if minute > 59:
minute = 0
day += 1
if month == 2 and day > 28:
day = 0
month = 3
if month == 3 and day > 31:
day = 0
month = 4
return day, month, minute, vial
for root, dirs, files in sorted(os.walk(root_folder)):
for filename in sorted(files):
if not filename.endswith(video_extension):
continue
bn = os.path.basename(filename)
date = bn.split('.')[0]
cur_day = int(date.split('-')[2])
cur_month = int(date.split('-')[1])
time = bn.split('.')[1]
cur_minute = int(time.split('-')[1])
cur_vial = int(bn.split('.')[2][1])
printed = False
cnt = 0
while day != cur_day \
or month != cur_month \
or minute != cur_minute \
or vial != cur_vial:
if not printed:
printed = True
# print day, month, minute, vial
cnt += 1
day, month, minute, vial = incrementTime(day, month, minute, vial)
if printed:
print filename, 'and next', cnt / 4, 'minutes'
day, month, minute, vial = incrementTime(day, month, minute, vial)
def checkVideoLenghtsInAnnotations(root_folder, pos_folder, anno_folder,
video_extension='mp4',
correct_video_length=False):
import pyTools.system.videoExplorer as VE
import numpy as np
import yaml
vE = VE.videoExplorer()
anno = Annotation()
unequalFiles = {}
for root, dirs, files in sorted(os.walk(root_folder)):
for filename in sorted(files):
if filename.endswith(video_extension):
anno_filename = filename[:-len(video_extension)] + 'csv'
pos_filename = filename[:-len(video_extension)] + 'pos.npy'
af = os.path.join(os.path.dirname(root),
anno_folder,
anno_filename)
pf = os.path.join(os.path.dirname(root),
pos_folder,
pos_filename)
videoLength = vE.retrieveVideoLength(os.path.join(root,
filename))
try:
posLength = len(np.load(pf))
except IOError:
print "No posfile for {}".format(pf)
continue
if posLength != (videoLength):
print "pos f**k:\nfile: {}\nvL: {}\npL: {}".format(filename,
videoLength,
posLength)
else:
print "pos good:\nfile: {}\nvL: {}\npL: {}".format(filename,
videoLength,
posLength)
try:
anno.loadFromFile(af)
except IOError:
print "No annofile for {}".format(af)
continue
annoLength = anno.getLength()
if annoLength != videoLength:
print "anno f**k:\nfile: {}\nvL: {}\naL: {}".format(filename,
videoLength,
annoLength)
if correct_video_length:
anno.setLength(videoLength)
anno.saveToFile(af)
unequalFiles[filename] = [videoLength, posLength, annoLength]
yamlStr = yaml.dump(unequalFiles, default_flow_style=False,
encoding=('utf-8'))
with open('~/Desktop/files.json', 'w') as f:
f.writelines(yamlStr)