def download_set(self, name):
""" download the given set
"""
self.root_export = join(self.root, 'marconi/' + name)
url = 'http://resources.mpi-inf.mpg.de/marconi/Data/' + name + '/Images.zip'
self.verbose = False
self.download_and_unzip(url,
zipfile_name='Images.zip',
dest_folder='marconi/' + name + "/Images",
dest_force=True,
root_folder='marconi/' + name)
# url = 'http://resources.mpi-inf.mpg.de/marconi/Data/' + name + '/CNN_Detections.zip'
# self.download_and_unzip(url, zipfile_name='CNN_Detections.zip',
# dest_folder='marconi/' + name + "/CNN_Detections", dest_force=True,
# root_folder='marconi/' + name)
self.root_export = join(self.root, 'marconi')
url = 'http://resources.mpi-inf.mpg.de/marconi/Data/' + name + \
'/Annotations.mat'
self.download_file(url, file_name='Annotations.mat', dest_folder=name)
url = 'http://resources.mpi-inf.mpg.de/marconi/Data/' + name + \
'/Calibration.dat'
self.download_file(url, file_name='Calibration.dat', dest_folder=name)
self.verbose = self._verbose
utils.talk("\t" + name + " finished", self._verbose)
def __init__(self, root, username, password, verbose=True):
"""
"""
utils.talk("UMPM", verbose)
data_root = join(root, 'umpm')
self.data_root = data_root
root_url = 'http://umpm-mirror.cs.uu.nl/download/'
if not isdir(data_root):
makedirs(data_root)
calib_zip = join(data_root, 'umpm_camcalib1.zip')
if not isfile(calib_zip):
calib_url = 'http://umpm-mirror.cs.uu.nl/download/umpm_camcalib1.zip'
download.download_with_login(calib_url, data_root, username,
password)
assert isfile(calib_zip)
calib_dir = join(data_root, 'Calib')
if not isdir(calib_dir):
unzip.unzip(calib_zip, data_root)
assert isdir(calib_dir)
for file in UMPM.get_file_list():
cur_loc = join(data_root, file)
fzip = join(cur_loc, file + ".zip")
cur_url = root_url + file + '.zip'
# fc3d_gt = join(data_root, file + '.c3d')
# cur_url_gt = root_url_gt + file + '.c3d'
if not isdir(cur_loc):
utils.talk("could not find location " + file, verbose)
if not isfile(fzip):
utils.talk("could not find file " + file + '.zip', verbose)
download.download_with_login(cur_url, cur_loc, username,
password)
if not isdir(join(cur_loc, 'Groundtruth')):
# is not unzipped
utils.talk("unzipping " + fzip, verbose)
unzip.unzip(fzip, cur_loc, del_after_unzip=True)
video_loc = join(cur_loc, 'Video')
lzma_videos = [
join(video_loc, f) for f in listdir(video_loc)
if f.endswith('.xz')
]
for lzma_video in lzma_videos:
utils.talk('unzipping video ' + lzma_video, verbose)
unzip.unzip(lzma_video, video_loc, del_after_unzip=True)
def get_raw(self, folder):
""" gets the raw identity pictures
"""
X_is_loaded = False
if self.memmapped:
fmmap = self.get_memmapped_file_name(folder)
data_shape = self.get_memmapped_file_shape(folder)
if isfile(fmmap):
X = np.memmap(fmmap, dtype='uint8', mode="r", shape=data_shape)
X_is_loaded = True
loc = join(self.root_export, folder)
imgs = sorted([f for f in listdir(loc) if isfile(join(loc, f)) and \
f.endswith('jpg')])
if not X_is_loaded:
force_shape = self.force_shape
if force_shape is None:
X = np.array([imread(join(loc, f)) for f in imgs], 'uint8')
else:
X = np.array([imresize(imread(join(loc, f)), size=force_shape) \
for f in imgs], 'uint8')
if self.memmapped:
# store to memmap
X_ = np.memmap(fmmap, dtype='uint8', mode="w+", shape=data_shape)
for i, x in enumerate(X):
X_[i] = x
utils.talk('flush memmaped ' + fmmap, self.verbose)
del X_ # flush
time.sleep(3)
X = np.memmap(fmmap, dtype='uint8', mode="r", shape=data_shape)
identities = np.array([int(f[0:2]) if f.startswith('-1') else int(f[0:4]) \
for f in imgs])
cameras = np.array([int(f[4]) if f.startswith('-1') else int(f[6]) \
for f in imgs])
sequences = np.array([int(f[6]) if f.startswith('-1') else int(f[8]) \
for f in imgs])
frames = np.array([int(f[8:14]) if f.startswith('-1') else int(f[10:16]) \
for f in imgs])
Y = np.vstack((identities, cameras, sequences, frames))
return X, np.rollaxis(Y, 1)
def get_train(self, folder, memmapped=False):
""" Gets the raw MOT data for training
"""
parent = 'train'
X, Y_det = MOT_X.get_raw(self,
folder,
parent=parent,
memmapped=memmapped)
# Y-gt
root = join(self.root_export, parent)
loc = join(root, folder)
gt_txt = join(join(loc, "gt"), 'gt.txt')
Y_gt = np.loadtxt(gt_txt, delimiter=',')
utils.talk(self.name + ' Y_gt loaded', self.verbose)
return X, Y_det, Y_gt
def get_raw(self, folder, parent, memmapped=False):
""" get the raw train data
"""
root = join(self.root_export, parent)
loc = join(root, folder)
# X
img_loc = join(loc, "img1")
imgs = sorted([join(img_loc, f) \
for f in listdir(img_loc) if isfile(join(img_loc, f))])
if memmapped:
if self.resize is not None:
raise Exception('Cannot resize and use memmap')
# we NEED the data shape!
num_frames = len(imgs)
w, h, c = imread(imgs[0]).shape
data_shape = (num_frames, w, h, c)
fmmap = join(loc, 'data.memmap')
if not isfile(fmmap):
# data has to be created
utils.talk(self.name + ": create memmapped file " + fmmap,
self.verbose)
X = np.memmap(fmmap,
dtype='uint8',
mode="w+",
shape=data_shape)
for i, f in enumerate(imgs):
X[i] = imread(f)
X.flush()
del X
utils.talk(self.name + ": load memmapped file " + fmmap,
self.verbose)
X = np.memmap(fmmap, dtype='uint8', mode='r', shape=data_shape)
else:
# Not memmapped files -> load everything into memory
if self.resize is None:
X = np.array([imread(f) for f in imgs], 'uint8')
else:
X = np.array(
[imresize(imread(f), size=self.resize) for f in imgs],
'uint8')
utils.talk(self.name + ' X loaded', self.verbose)
# Y-det
det_txt = join(join(loc, "det"), 'det.txt')
Y_det = np.loadtxt(det_txt, delimiter=',')
utils.talk(self.name + ' Y_det loaded', self.verbose)
return X, Y_det
def __init__(self, root, username, pw):
"""
:param root:
:param username: to access the data a username and
:param pw: password are required
"""
data_root = join(root, 'celltrackingchallenge')
self.data_root = data_root
test_root = join(data_root, 'test')
train_root = join(data_root, 'train')
if not isdir(test_root):
makedirs(test_root)
if not isdir(train_root):
makedirs(train_root)
Train_urls = CellTrackingChallenge2D.get_training_dataset_urls()
Test_urls = CellTrackingChallenge2D.get_test_dataset_urls()
for url_train, url_test in zip(Train_urls, Test_urls):
fname = utils.get_filename_from_url(url_test)
assert fname == utils.get_filename_from_url(url_train)
fzip_train = join(train_root, fname)
fzip_test = join(test_root, fname)
if not isfile(fzip_train):
utils.talk(
"could not find train file:" + fname + ", .. downloading",
True)
download.download_with_login(url_train, train_root, username,
pw)
if not isfile(fzip_test):
utils.talk(
"could not find test file:" + fname + ", .. downloading",
True)
download.download_with_login(url_test, test_root, username, pw)
data_name = fname[:-4]
cur_train_loc = join(train_root, data_name)
cur_test_loc = join(test_root, data_name)
if not isdir(cur_train_loc):
utils.talk(
"Could not find folder " + cur_train_loc + ', .. unzip',
True)
unzip.unzip(fzip_train, train_root)
if not isdir(cur_test_loc):
utils.talk(
"Could not find folder " + cur_test_loc + ', .. unzip',
True)
unzip.unzip(fzip_test, test_root)
def __init__(self, root, verbose=True):
Dataset.__init__(self, 'marconi', root, verbose)
utils.talk('(MARCOnl)', verbose)
# This is a horrible hack but it seems that
# the python tools cannot unzip certain zip files
# (that seem to be zipped on OSX) and I have no
# time to find a 'good' solution so the ugly hack
# is: use the system tools which for some reason
# work! (on LINUX!!) --> see MARCOnI
self.force_unzip_with_os_tools = True
self._verbose = verbose
self.root_export = join(root, 'marconi')
self.download_set("Soccer")
self.download_set("Kickbox")
self.download_set("SBoard")
self.download_set("Soccer2")
self.download_set("Walk1")
self.download_set("Walk2")
self.download_set("Volleyball")
self.download_set("Juggling")
self.download_set("Run2")
def unzip_using_os_tools(fzip, export_folder, verbose):
""" unzips the given file to the given folder using os tools
"""
#TODO: this has to be figured out eventually
command = ["unzip", fzip, '-d', export_folder]
utils.talk('\ttry unzip using OS tools', verbose)
process = subprocess.Popen(command, stdout=subprocess.PIPE)
utils.talk('\tunzip using OS tools', verbose)
output, err = process.communicate()
utils.talk('\terr:\t' + str(err), verbose)
def unzip(self):
""" Unzip the name file into the root_export directory
If the zip file is not found an exception is thrown
"""
dest = join(self.root, self.name)
if not exists(dest):
fzip = join(self.root, self.name + ".zip")
if not isfile(fzip):
utils.talk('Could not find ' + fzip, self.verbose)
raise Exception('Could not find ' + fzip)
zip_ref = zipfile.ZipFile(fzip, 'r')
utils.talk("unzip " + fzip + " -> " + self.root, self.verbose)
zip_ref.extractall(self.root_export)
zip_ref.close()
else:
utils.talk(dest + ' found :)', self.verbose)
def download_file(self, url, file_name, dest_folder=None):
""" Only download the file
"""
if dest_folder is None:
dest = join(self.root_export, self.name)
else:
dest = join(self.root_export, dest_folder)
if not exists(dest):
makedirs(dest)
fname = join(dest, file_name)
if not isfile(fname):
utils.talk("Could not find " + fname + " ..., downloading",
self.verbose)
with urllib.request.urlopen(url) as res, open(fname, 'wb') as f:
utils.talk(url + " downloaded..", self.verbose)
shutil.copyfileobj(res, f)
else: # file exists
utils.talk("File " + fname + " found :)", self.verbose)
def unzip(fzip,
export_folder,
verbose=False,
force_os_tools=False,
del_after_unzip=False):
"""
@param fzip: {String} full path to the zip-file
@param export_folder: {String} place to unzip
@param verbose: {boolean} if true: debug
@param force_os_tools: {boolean} if true: try to use OS tools
This is a horrible hack but it seems that
the python tools cannot unzip certain zip files
(that seem to be zipped on OSX) and I have no
time to find a 'good' solution so the ugly hack
is: use the system tools which for some reason
work! (on LINUX!!)
@param del_after_unzip: delete after unzipping
"""
if force_os_tools:
assert has_os_unzip()
unzip_using_os_tools(fzip, export_folder, verbose)
else:
if fzip.endswith('.zip'):
try:
utils.talk("unzip " + fzip + " -> " + export_folder, verbose)
zip_ref = zipfile.ZipFile(fzip, 'r')
zip_ref.extractall(export_folder)
zip_ref.close()
except zipfile.BadZipFile as e:
utils.talk('unzip exception:' + str(e), verbose)
# -- try it the hard way -- use OS tools
if has_os_unzip():
unzip_using_os_tools(fzip, export_folder, verbose)
else:
raise
elif fzip.endswith('tar.gz'):
mode = 'r:gz'
utils.talk("untar " + fzip + " -> " + export_folder, verbose)
tar = tarfile.open(fzip, mode)
tar.extractall(export_folder)
tar.close()
elif fzip.endswith('tar.bz2'):
command = ['tar', 'xvjf', fzip]
utils.talk('\ttry to untar using OS tools', verbose)
process = subprocess.Popen(command, stdout=subprocess.PIPE)
utils.talk('\tuntar using OS tools', verbose)
output, err = process.communicate()
process.wait()
utils.talk('\tuntar finished', verbose)
utils.talk('\t\terrors:\t' + str(err), verbose)
elif fzip.endswith('.xz'):
command = ["unxz", fzip]
utils.talk('\ttry unxz using OS tools', verbose)
process = subprocess.Popen(command, stdout=subprocess.PIPE)
utils.talk('\tunxz using OS tools', verbose)
output, err = process.communicate()
process.wait()
utils.talk('\terr:\t' + str(err), verbose)
else:
raise RuntimeError("No unzip routine found for " + fzip)
if del_after_unzip:
time.sleep(0.5) # just to be sure sleep some time
if isfile(fzip):
os.remove(fzip)
def get_raw(self, folder):
# step 1: check if we already have memmapped file:
X_is_loaded = False
if self.memmapped:
fmmap = self.get_memmapped_file_name(folder)
if isfile(fmmap):
utils.talk('found memmaped ' + fmmap, self.verbose)
data_shape = self.get_memmapped_file_shape(folder)
X = np.memmap(fmmap, dtype='uint8', mode="r", shape=data_shape)
X_is_loaded = True
else:
utils.talk('could not find memmaped ' + fmmap, self.verbose)
Y_is_loaded = False
fydata = self.get_y_file(folder)
if isfile(fydata):
Y = np.load(fydata)
Y_is_loaded = True
if X_is_loaded and Y_is_loaded:
return X, Y
f = h5py.File(self.hd5file, 'r+')
data = f[folder]
_, n = data.shape
tw = self.target_w
th = self.target_h
current_id = 1
Imgs = []
Ids = []
for view in range(n):
utils.talk("cuhk03: view \"" + folder + "\" " + \
str(view+1) + "/" + str(n), self.verbose)
V = f[data[0][view]]
ten, M = V.shape # M is number of identities
for i in range(M):
for j in range(ten):
img = f[V[j][i]].value
if len(img.shape) == 3:
if not X_is_loaded:
img = np.swapaxes(img, 0, 2)
img = resize(img, (th, tw), mode='constant')
Imgs.append((img * 255).astype('uint8'))
Ids.append(current_id)
current_id += 1
if not X_is_loaded:
if self.memmapped:
utils.talk('write memmaped ' + fmmap, self.verbose)
data_shape = self.get_memmapped_file_shape(folder)
X_ = np.memmap(fmmap,
dtype='uint8',
mode="w+",
shape=data_shape)
for i, x in enumerate(Imgs):
X_[i] = x
utils.talk('flush memmaped ' + fmmap, self.verbose)
del X_ # flush
time.sleep(3)
X = np.memmap(fmmap, dtype='uint8', mode="r", shape=data_shape)
else:
X = np.array(Imgs)
Y = np.array(Ids, 'int32')
if not Y_is_loaded:
np.save(fydata, Y)
return X, Y
def get_raw(self, folder):
""" gets the raw identity pictures
"""
loc = join(self.root_export, folder)
imgs = sorted([f \
for f in listdir(loc) if isfile(join(loc, f)) and \
f.endswith('jpg')])
X_is_loaded = False
is_memmapped = self.can_be_memmapped() and self.memmapped
if is_memmapped:
fmmap = self.get_memmapped_file_name(folder)
data_shape = self.get_memmapped_file_shape(folder)
if isfile(fmmap):
utils.talk('load memmap ' + fmmap, self.verbose)
X = np.memmap(fmmap, dtype='uint8', mode="r", shape=data_shape)
X_is_loaded = True
Y_is_loaded = False
fydata = self.get_y_file(folder)
if isfile(fydata):
Y = np.load(fydata)
Y_is_loaded = True
if X_is_loaded and Y_is_loaded:
return X, Y
if not X_is_loaded:
force_shape = self.force_shape
if force_shape is None:
X = np.array([imread(join(loc, f)) for f in imgs])
else:
X = np.array([imresize(imread(join(loc, f)), size=force_shape) \
for f in imgs])
if is_memmapped:
# store X to file
utils.talk('create memmap ' + fmmap, self.verbose)
X_ = np.memmap(fmmap,
dtype='uint8',
mode="w+",
shape=data_shape)
for i, x in enumerate(X):
X_[i] = x
utils.talk('flush memmaped ' + fmmap, self.verbose)
del X_ # flush
time.sleep(3)
X = np.memmap(fmmap, dtype='uint8', mode="r", shape=data_shape)
if not Y_is_loaded:
identities = np.array([int(f[0:4]) for f in imgs])
cameras = np.array([int(f[6]) for f in imgs])
frames = np.array([int(f[9:15]) for f in imgs])
Y = np.vstack((identities, cameras, frames))
Y = np.rollaxis(Y, 1)
np.save(fydata, Y)
return X, Y
def download_and_unzip(self,
url,
zipfile_name=None,
dest_folder=None,
dest_force=True,
root_folder=None):
""" Downloads and unzips a zipped data file
"""
if dest_folder is None:
dest = join(self.root, self.name)
else:
dest = join(self.root, dest_folder)
if dest_force:
export_folder = dest
else:
self.root_export
if root_folder is None:
root = self.root
else:
root = join(self.root, root_folder)
if not exists(root):
makedirs(root)
if not exists(dest):
utils.talk("could not find folder " + dest + "...", self.verbose)
if zipfile_name is None:
fzip = join(root, self.name + ".zip")
else:
fzip = join(root, zipfile_name)
if isfile(fzip):
utils.talk('found ' + fzip, self.verbose)
else:
utils.talk("could not find file " + fzip, self.verbose)
utils.talk("download from " + url, self.verbose)
with urllib.request.urlopen(url) as res, open(fzip, 'wb') as f:
utils.talk(url + " downloaded..", self.verbose)
shutil.copyfileobj(res, f)
unzip.unzip(fzip, export_folder, self.verbose,
self.force_unzip_with_os_tools)
else:
utils.talk(dest + ' found :)', self.verbose)
def get_raw(self, config=EgoHands_config.Polygon, memmapped=False):
"""
"""
# step 1: get all videos
labelled_samples_url = join(self.root_export, '_LABELLED_SAMPLES')
all_videos = [join(labelled_samples_url, f) for f in \
listdir(labelled_samples_url) if isdir(join(labelled_samples_url, f))]
# step 2: load video frames and polygon dataset
Y = []
if memmapped:
X_shape = (48, 100, 720, 1280, 3)
fmmap = join(self.root_export, 'egohands.memmap')
fmmap_exists = isfile(fmmap)
if not fmmap_exists:
X = np.memmap(fmmap, dtype='uint8', mode='w+', shape=X_shape)
else:
X = []
for vindx, vurl in enumerate(all_videos):
imgs = sorted([f \
for f in listdir(vurl) if isfile(join(vurl, f)) and \
f.endswith('jpg')])
assert (len(imgs) == 100) # sanity check
if memmapped:
if not fmmap_exists:
# if we already created the memmap file we do NOT
# want to recreate it!
imgs = np.array([imread(join(vurl, f)) for f in imgs], \
'uint8')
X[vindx] = imgs
else:
imgs = np.array([imread(join(vurl, f)) for f in imgs], 'uint8')
X.append(imgs)
polygon_url = join(vurl, 'polygons.mat')
M = loadmat(polygon_url)['polygons'][0]
Y_single_video = []
for i in range(100):
V = M[i]
Y_single_frame = []
for hand in range(4):
H = V[hand]
#if len(H) > 0:
if config is EgoHands_config.Polygon:
Y_single_frame.append(H)
elif len(H) > 1: # meaning: hand is not visible
x = H[:, 0]
y = H[:, 1]
top_right = (np.max(x), np.max(y))
bottom_left = (np.min(x), np.min(y))
Y_single_frame.append((top_right, bottom_left))
Y_single_video.append(Y_single_frame)
Y.append(Y_single_video)
# step 2: read metadata
#M = loadmat(join(self.root_export, 'metadata.mat'))
#
# M = loadmat(join(self.root_export, '_LABELLED_SAMPLES/CARDS_COURTYARD_B_T/polygons.mat'))
#
# X = imread(join(labelled_samples_url, 'CARDS_COURTYARD_B_T/frame_0011.jpg'))
if memmapped:
if not fmmap_exists:
#del X # flush the file
utils.talk('flush memmap to file', self.verbose)
X.flush() # write memmap to files
del X
X = np.memmap(fmmap, dtype='uint8', mode='r', shape=X_shape)
else:
X = np.array(X, 'uint8')
return X, np.array(Y)