def take(self, n):
S = np.random.choice(self.subjects(), n)
takelist = []
for d in dirlist(os.path.join(self.datadir, 'images')):
if filebase(d) in S:
f = np.random.choice(imlist(d), 1)[0]
im = ImageDetection(filename=f, category=filebase(d))
im = im.boundingbox(xmin=float(im.width() - 256) / 2.0,
ymin=float(im.height() - 256.0) / 2.0,
xmax=256.0 + ((im.width() - 256.0) / 2.0),
ymax=256.0 + ((im.height() - 256.0) / 2.0))
im = im.boundingbox(dilate=0.875) # central 224x224
takelist.append(im)
return takelist
def subjects(self):
if self._subjects is None:
self._subjects = [
filebase(f)
for f in txtlist(os.path.join(self.datadir, 'data'))
]
return self._subjects # cached
def subjects(self):
if self._subjects is None:
self._subjects = [
filebase(d)
for d in dirlist(os.path.join(self.datadir, 'images'))
]
return self._subjects # cached
def subjectset(self, wordnetid):
"""Iterator for single subject"""
assert wordnetid in self.wordnetid_to_name().keys(
), 'Invalid wordnetid "%s"' % wordnetid
d = os.path.join(self.datadir, wordnetid)
for f in imlist(d):
yield ImageDetection(filename=f, category=filebase(d))
def take_per_subject(self, n):
"""Randomly select n images per subject from the dataset"""
subjectid = self.subjects()
takelist = []
for s in subjectid:
d = os.path.join(self.datadir, s)
for k in range(0, n):
f = np.random.choice(imlist(d), 1)[0]
im = ImageDetection(filename=f).category(filebase(d))
takelist.append(im)
return takelist
def dataset(self):
"""Return a generator to iterate over dataset"""
for d in dirlist(os.path.join(self.datadir, 'images')):
for f in imlist(d):
im = ImageDetection(filename=f, category=filebase(d))
im = im.boundingbox(xmin=float(im.width() - 256) / 2.0,
ymin=float(im.height() - 256.0) / 2.0,
xmax=256.0 + ((im.width() - 256.0) / 2.0),
ymax=256.0 + ((im.height() - 256.0) / 2.0))
im = im.boundingbox(dilate=0.875) # central 224x224
yield im
def take(self, n, wordnetid=None):
"""Randomly select n images from the dataset, or n images of a given subjectid"""
subjectid = np.random.choice(
self.subjects(), n) if wordnetid is None else [wordnetid] * n
takelist = []
for s in subjectid:
d = os.path.join(self.datadir, s)
f = np.random.choice(imlist(d), 1)[0]
im = ImageDetection(filename=f).category(filebase(d))
takelist.append(im)
return takelist
def download(vidurl,
vidfile,
skip=False,
writeurlfile=True,
max_filesize='350m',
remove_parts=True,
verbose=False):
"""Use youtube-dl to download a video URL to a video file"""
user_agent = random.choice(complete_user_agents)
ydl_exe = os.path.join(filepath(sys.executable), 'youtube-dl')
if not os.path.exists(ydl_exe):
raise ImportError(
'Optional package "youtube-dl" not installed - Run "pip install youtube-dl"'
)
try:
print('[vipy.videosearch.download]: saving "%s" to "%s"' %
(vidurl, vidfile))
for f in glob.glob("%s*" % vidfile):
os.remove(f) # youtube-dl will not overwrite, so we force it
cmd = '%s %s "%s" -o %s --max-filesize %s --no-check-certificate --user-agent="%s"' % (
ydl_exe, '-q' if not verbose else '', vidurl, vidfile,
max_filesize, user_agent) # must be on path
if verbose:
print('[vipy.videosearch.download]: executing \'%s\'' % cmd)
erno = os.system(cmd)
if erno != 0:
raise ValueError('youtube-dl returned %d' % erno)
if os.path.isfile(vidfile):
if writeurlfile:
urlfile = os.path.join(filepath(vidfile),
'%s.url' % filebase(vidfile))
with open(urlfile, 'w') as f:
f.write(vidurl + '\n')
if remove_parts and os.path.isfile(vidfile + '.part'):
partfile = vidfile + '.part'
print('[vipy.youtube.download]: removing partial file: %s' %
partfile)
os.remove(partfile)
except KeyboardInterrupt:
raise
except Exception as exception:
print(exception)
# http://rg3.github.io/youtube-dl/supportedsites.html
print('[vipy.videosearch.download]: download failed - skipping')
return None
if erno == 256:
raise KeyboardInterrupt
elif skip and erno: # use this if every video on a page fails
return None
else:
return vidfile
def _trainset(self):
"""Save a csv file containing each image on a line for Megaface_Challenge_1M_disjoint_LOOSE.tar.gz"""
outfile = os.path.join(self.datadir,
'Megaface_Challenge_1M_disjoint_LOOSE.csv')
subdir = os.path.join(self.datadir,
'Megaface_Challenge_1M_disjoint_LOOSE')
D = dirlist(subdir)
filelist = []
for (k, d) in enumerate(D):
print('[MF2.trainset][%d/%d]: creating image list for "%s"' %
(k, len(D), d))
for f in imlist(d):
filelist.append((f, filebase(d)))
return writecsv(filelist, outfile)
def dataset(self):
"""Return a generator to iterate over dataset"""
SCHEMA = [
'id', 'url', 'left', 'top', 'right', 'bottom', 'pose',
'detection_score', 'curation'
]
for f in txtlist(os.path.join(self.datadir, 'files')):
for r in readcsv(f, separator=' '):
im = ImageDetection(url=r[2],
category=filebase(f),
xmin=float(r[3]),
ymin=float(r[4]),
xmax=float(r[5]),
ymax=float(r[6]),
attributes=dict(zip(SCHEMA, r)))
yield im
def tinyset(self, size=1000):
"""Return the first (size) image objects in the dataset"""
outlist = []
imglist = np.random.permutation(
[f[0] for f in readcsv(self._imagelist())])
for (k, f) in enumerate(imglist):
print('[megaface.dataset][%d/%d]: importing "%s"' % (k, size, f))
A = self._attributes(os.path.join(self.datadir, f))
outlist = outlist + [
ImageDetection(filename=os.path.join(self.datadir, f),
category=filebase(f)).boundingbox(
xmin=A['bounding_box']['x'],
ymin=A['bounding_box']['y'],
width=A['bounding_box']['width'],
height=A['bounding_box']['height'])
]
if k > size:
break
return outlist
def take(self, n):
"""Randomly select n frames from dataset"""
takelist = []
SCHEMA = [
'id', 'url', 'left', 'top', 'right', 'bottom', 'pose',
'detection_score', 'curation'
]
for csvfile in np.random.choice(
txtlist(os.path.join(self.datadir, 'data')), n):
csv = readcsv(csvfile, separator=' ')
r = csv[np.random.randint(1, len(csv))] # not including header
im = ImageDetection(url=r[2],
category=filebase(csvfile),
xmin=float(r[3]),
ymin=float(r[4]),
xmax=float(r[5]),
ymax=float(r[6]),
attributes=dict(zip(SCHEMA, r)))
takelist.append(im)
return takelist
def tinyset(self, size=1000):
"""Return the first (size) image objects in the trainset"""
outlist = []
if not os.path.exists(
os.path.join(self.datadir,
'Megaface_Challenge_1M_disjoint_LOOSE.csv')):
print('[MF2.tinyset]: generating csv file for MF2')
self._trainset()
imglist = np.random.permutation([
f[0] for f in readcsv(
os.path.join(self.datadir,
'Megaface_Challenge_1M_disjoint_LOOSE.csv'))
])
for (k, f) in enumerate(imglist):
print('[MF2.tinyset][%d/%d]: importing "%s"' % (k, size, f))
outlist = outlist + [
ImageDetection(filename=os.path.join(self.datadir, f),
category=filebase(f))
]
if k > size:
break
return outlist
def editedat(self):
"""Android appends an '_<int>' timestamp as milliseconds since epoch (POSIX timestamp), iOS will replace the first '_datetimestr' with a new datetimest"""
return filebase(
self._jsonfile).split("_")[-1] if self.isedited() else None
def isedited(self):
return ("_" in self._jsonfile
and filebase(self._jsonfile).split("_")[0] == self.videoid()
) # edited JSON has the structure $VIDEOID_TIMESTAMP.json
def _parse(self):
outlist = []
id2name = {k:v for (k,v) in readcsv(os.path.join(self.datadir, 'names.txt'), separator=' ')}
for d in dirlist(self.datadir):
outlist = outlist + [ImageDetection(filename=imfile, category=id2name[str(filebase(d))], xmin=13, ymin=13, xmax=250 - 13, ymax=250 - 13) for imfile in imlist(d)]
return outlist
def subjects(self):
if self._subjects is None:
self._subjects = [filebase(d) for d in dirlist(self.datadir)]
return self._subjects # cached
def dataset(self):
"""Return a generator to iterate over dataset"""
for d in dirlist(os.path.join(self.datadir)):
for f in imlist(d):
yield ImageDetection(filename=f).category(filebase(d))
def __init__(self,
videodir,
repodir,
contrib=False,
stride=1,
verbose=True,
n_videos=None,
d_category_to_shortlabel=None):
"""Parse MEVA annotations (http://mevadata.org) for KNown Facility 1 dataset into vipy.video.Scene() objects
Kwiver packet format: https://gitlab.kitware.com/meva/meva-data-repo/blob/master/documents/KPF-specification-v4.pdf
Inputs:
-videodir=str: path to Directory containing 'drop-01'
-repodir=str: path to directory containing clone of https://gitlab.kitware.com/meva/meva-data-repo
-stride=int: the temporal stride in frames for importing bounding boxes, vipy will do linear interpoluation and boundary handling
-n_videos=int: only return an integer number of videos, useful for debugging or for previewing dataset
-contrib=bool: include the noisy contrib anntations from DIVA performers
-d_category_to_shortlabel is a dictionary mapping category names to a short displayed label on the video. The standard for visualization is that
tracked objects are displayed with their category label (e.g. 'Person', 'Vehicle'), and activities are labeled according to the set of objects that
performing the activity. When an activity occurs, the set of objects are labeled with the same color as 'Noun Verbing' (e.g. 'Person Entering',
'Person Reading', 'Vehicle Starting') where 'Verbing' is provided by the shortlabel. This is optional, and will use the default mapping if None
-verbose=bool: Parsing verbosity
"""
self.videodir = videodir
self.repodir = repodir
assert os.path.exists(
os.path.join(self.videodir, 'drop-01')
), "Invalid input - videodir must contain the drop-01, drop-02 and drop-03 subdirectories. See http://mevadata.org/#getting-data"
assert os.path.exists(
os.path.join(self.repodir, 'annotation')
), "Invalid input - repodir must contain the clone of https://gitlab.kitware.com/meva/meva-data-repo"
self._d_category_to_shortlabel = {
'person_abandons_package': 'Abandoning',
'person_closes_facility_door': 'Closing',
'person_closes_trunk': 'Closing trunk',
'person_closes_vehicle_door': 'Closing door',
'person_embraces_person': 'Hugging',
'person_enters_scene_through_structure': 'Entering',
'person_enters_vehicle': 'Entering',
'person_exits_scene_through_structure': 'Exiting',
'person_exits_vehicle': 'Exiting',
'hand_interacts_with_person': 'Using hand',
'person_carries_heavy_object': 'Carrying',
'person_interacts_with_laptop': 'Using laptop',
'person_loads_vehicle': 'Loading',
'person_transfers_object': 'Transferring',
'person_opens_facility_door': 'Opening door',
'person_opens_trunk': 'Opening trunk',
'person_opens_vehicle_door': 'Opening door',
'person_talks_to_person': 'Talking',
'person_picks_up_object': 'Picking up',
'person_purchases': 'Purchasing',
'person_reads_document': 'Reading',
'person_rides_bicycle': 'Riding',
'person_puts_down_object': 'Putting down',
'person_sits_down': 'Sitting',
'person_stands_up': 'Standing',
'person_talks_on_phone': 'Talking',
'person_texts_on_phone': 'Texting',
'person_steals_object': 'Stealing',
'person_unloads_vehicle': 'Unloading',
'vehicle_drops_off_person': 'Dropping off',
'vehicle_picks_up_person': 'Picking up',
'vehicle_reverses': 'Reversing',
'vehicle_starts': 'Starting',
'vehicle_stops': 'Stopping',
'vehicle_turns_left': 'Turning left',
'vehicle_turns_right': 'Turning right',
'vehicle_makes_u_turn': 'Turning around'
}
self._d_category_to_shortlabel = {
k: v.lower()
for (k, v) in self._d_category_to_shortlabel.items()
}
self._d_oldcategory_to_shortlabel = {
'Closing_Trunk': 'Closing',
'Open_Trunk': 'Opening',
'Riding': 'Riding',
'Talking': 'Talking',
'person_talks_to_person': 'Talking',
'Transport_HeavyCarry': 'Carrying',
'Unloading': 'Unloading',
'abandon_package': 'Abandoning',
'hand_interaction': 'Using Hand',
'object_transfer': 'Transferring',
'person_closes_facility_door': 'Closing',
'person_closes_vehicle_door': 'Closing',
'person_enters_through_structure': 'Entering',
'person_enters_vehicle': 'Entering',
'person_exits_through_structure': 'Exiting',
'person_exits_vehicle': 'Exiting',
'person_laptop_interaction': 'Interacting',
'person_loads_vehicle': 'Loading',
'person_opens_facility_door': 'Opening',
'person_opens_vehicle_door': 'Opening',
'person_person_embrace': 'Hugging',
'person_picks_up_object': 'Picking up',
'person_purchasing': 'Purchasing',
'person_reading_document': 'Reading',
'person_sets_down_object': 'Setting down',
'person_sitting_down': 'Sitting',
'person_standing_up': 'Standing',
'person_stands_up': 'Standing',
'specialized_talking_phone': 'Talking',
'specialized_texting_phone': 'Texting',
'theft': 'Theft',
'vehicle_drops_off_person': 'Dropping off',
'vehicle_picks_up_person': 'Picking up',
'vehicle_reversing': 'Reversing',
'vehicle_starting': 'Starting',
'vehicle_stopping': 'Stopping',
'vehicle_turning_left': 'Turning left',
'vehicle_turning_right': 'Turning right',
'vehicle_u_turn': 'Turning around'
}
self._d_oldcategory_to_newcategory = {
k: v
for (k, v) in readcsv(
os.path.join(self.repodir, 'documents',
'activity-name-mapping.csv'))[1:]
}
d_category_to_shortlabel = d_category_to_shortlabel if d_category_to_shortlabel is not None else self._d_category_to_shortlabel
d_videoname_to_path = {filebase(f): f for f in self._get_videos()}
yamlfiles = zip(self._get_types_yaml(), self._get_geom_yaml(),
self._get_activities_yaml())
yamlfiles = [
y for y in yamlfiles if contrib is True or 'contrib' not in y[0]
]
yamlfiles = list(
yamlfiles)[0:n_videos] if n_videos is not None else list(yamlfiles)
if verbose:
print('[vipy.dataset.meva.KF1]: Loading %d YAML files' %
len(yamlfiles))
if len(yamlfiles) > 100 and vipy.globals.num_workers() == 1:
print(
'[vipy.dataset.meva.KF1]: This takes a while since parsing YAML files in python is painfully slow, consider calling "vipy.globals.num_workers(8)" before loading the dataset for parallel parsing'
)
if vipy.globals.num_workers() > 1:
from vipy.batch import Batch
self._vidlist = Batch(list(yamlfiles)).map(
lambda tga: self._parse_video(d_videoname_to_path,
d_category_to_shortlabel,
tga[0],
tga[1],
tga[2],
stride=stride,
verbose=verbose))
else:
self._vidlist = [
self._parse_video(d_videoname_to_path,
d_category_to_shortlabel,
t,
g,
a,
stride=stride,
verbose=verbose) for (t, g, a) in yamlfiles
]
self._vidlist = [v for v in self._vidlist if v is not None]
def fastset(self):
"""Return a generator to iterate over dataset"""
for d in dirlist(os.path.join(self.datadir, 'images')):
for f in imlist(d):
im = ImageDetection(filename=f, category=filebase(d))
yield im
def _parse_video(self,
d_videoname_to_path,
d_category_to_shortlabel,
types_yamlfile,
geom_yamlfile,
act_yamlfile,
stride=1,
verbose=False):
"""Reference: https://gitlab.kitware.com/meva/meva-data-repo/-/blob/master/documents/KPF-specification-v4.pdf"""
# Read YAML
if verbose:
print('[vipy.dataset.meva.KF1]: Parsing "%s"' % (act_yamlfile))
geom_yaml = readyaml(geom_yamlfile)
types_yaml = readyaml(types_yamlfile)
act_yaml = readyaml(act_yamlfile)
# Sanity check
assert act_yamlfile.split('.')[:-2] == geom_yamlfile.split(
'.')[:-2], "Unmatched activity and geom yaml file"
assert len(
set([
types_yaml[0]['meta'], geom_yaml[0]['meta'],
act_yaml[0]['meta']
])) == 1, "Mismatched video name for '%s'" % act_yamlfile
videoname = act_yaml[0]['meta'] if act_yaml[0]['meta'][
-4:] != '.avi' else act_yaml[0]['meta'][0:-4] # strip .avi
if videoname not in d_videoname_to_path:
if verbose:
print(
'[vipy.dataset.meva.KF1]: Invalid MEVA video "%s" in "%s" - Ignoring'
% (videoname, filebase(act_yamlfile)))
return None
# Parse video
framerate = 30.0 # All videos are universally 30Hz (from Roddy)
vid = Scene(filename=d_videoname_to_path[videoname],
framerate=framerate)
# Parse tracks
d_id1_to_category = {}
for t in types_yaml:
if 'types' in t:
d_id1_to_category[t['types']['id1']] = list(
t['types']['cset3'].keys())[0]
d_id1_to_track = {}
d_geom_yaml = groupbyasdict(
[x['geom'] for x in geom_yaml if 'geom' in x], lambda v: v['id1'])
assert stride >= 1, "Invalid stride"
for (id1, geom_yaml) in d_geom_yaml.items():
geom_yaml = sorted(geom_yaml,
key=lambda x: int(x['ts0'])) # increasing
for (k_geom, v) in enumerate(geom_yaml):
if stride > 1 and k_geom > 0 and (
k_geom <
(len(geom_yaml) - stride)) and (k_geom % stride != 0):
continue # Use vipy track interpolation to speed up parsing
keyframe = int(v['ts0'])
bb = [int(x) for x in v['g0'].split(' ')]
bbox = BoundingBox(xmin=bb[0],
ymin=bb[1],
xmax=bb[2],
ymax=bb[3])
if not bbox.isvalid():
if verbose:
print(
'[vipy.dataset.meva.KF1]: Invalid bounding box: id1=%s, bbox="%s", file="%s" - Ignoring'
% (str(v['id1']), str(bbox),
delpath(self.repodir, geom_yamlfile)))
elif v['id1'] not in d_id1_to_track:
d_id1_to_track[v['id1']] = Track(
category=d_id1_to_category[v['id1']],
framerate=framerate,
keyframes=[keyframe],
boxes=[bbox],
boundary='strict')
else:
d_id1_to_track[v['id1']].add(keyframe=keyframe, box=bbox)
# Add tracks to scene
for (k, v) in d_id1_to_track.items():
try:
vid.add(
v, rangecheck=True
) # throw exception if all tracks are outside the image rectangle
except Exception as e:
print(
'[vipy.dataset.meva.KF1]: track import error "%s" for trackid=%s, track=%s - SKIPPING'
% (str(e), k, str(v)))
# Parse activities
for v in act_yaml:
if 'act' in v:
if 'act2' in v['act']:
act2 = v['act']['act2']
if isinstance(act2, set):
category = list(act2)[0]
elif isinstance(act2, dict):
category = list(act2.keys())[0]
else:
raise ValueError('YAML parsing error for "%s"' %
str(act2))
elif 'act3' in v['act']:
act3 = v['act']['act3']
if isinstance(act3, set):
category = list(act3)[0]
elif isinstance(act3, dict):
category = list(act3.keys())[0]
else:
raise ValueError('YAML parsing error for "%s"' %
str(act3))
else:
raise ValueError(
'Invalid activity YAML - act2 or act3 must be specified'
)
assert len(
v['act']
['timespan']) == 1, "Multi-span activities not parsed"
if category not in self.categories():
if category in self._d_oldcategory_to_newcategory:
category = self._d_oldcategory_to_newcategory[
category] # rationalize
else:
raise ValueError('undefined category "%s"' % category)
startframe = int(v['act']['timespan'][0]['tsr0'][0])
endframe = int(v['act']['timespan'][0]['tsr0'][1])
actorid = [x['id1'] for x in v['act']['actors']]
for aid in actorid:
if not aid in d_id1_to_track:
print(
'[vipy.dataset.meva.KF1]: ActorID %d referenced in activity yaml "%s" not found in geom yaml "%s" - Skipping'
% (aid, delpath(self.repodir, act_yamlfile),
delpath(self.repodir, geom_yamlfile)))
# Add activity to scene: include YAML file details in activity attributes for provenance if there are labeling bugs
tracks = {
d_id1_to_track[aid].id(): d_id1_to_track[aid]
for aid in actorid if aid in d_id1_to_track
}
if len(tracks) > 0:
try:
vid.add(Activity(
category=category,
shortlabel=d_category_to_shortlabel[category],
startframe=startframe,
endframe=endframe,
tracks=tracks,
framerate=framerate,
attributes={
'act': v['act'],
'act_yaml': act_yamlfile,
'geom_yaml': geom_yamlfile
}),
rangecheck=True)
except Exception as e:
print(
'[vipy.dataset.meva.KF1]: activity import error "%s" for activity="%s" - SKIPPING'
% (str(e), str(v)))
return vid
def by_subject(self, wordnetid):
d = os.path.join(self.datadir, 'images', wordnetid)
for f in imlist(d):
yield ImageDetection(filename=f, category=filebase(d))
