def prepare_cpuvisor(base_path, component_cfgs):
component_paths = component_cfgs['components']
links = component_cfgs['links']
collection = component_cfgs['collection']
index_dir = os.path.join(collection['paths']['index_data'], 'cpuvisor-srv')
templates_dir = os.path.join('templates', 'cpuvisor-srv')
path = component_paths['cpuvisor-srv']
cpuvisortls = utils.import_python_module_from_path(
component_paths['cpuvisor-srv'], 'download_data')
if not os.path.exists(os.path.join(component_paths['cpuvisor-srv'],
'bin')):
raise RuntimeError(
'[cpuvisor] Missing bin directory - compile before running link_components!'
)
# prepare endpoints and paths
models_path = os.path.join(component_paths['cpuvisor-srv'], 'model_data')
negimgs_path = os.path.join(component_paths['cpuvisor-srv'], 'server_data',
'neg_images')
negidx_path = os.path.join(component_paths['cpuvisor-srv'], 'negpaths.txt')
negfeats_path = os.path.join(component_paths['cpuvisor-srv'],
'server_data', 'negfeats.binaryproto')
server_endpoint = 'tcp://127.0.0.1:%d' % links['cpuvisor-srv'][
'server_port']
notify_endpoint = 'tcp://127.0.0.1:%d' % links['cpuvisor-srv'][
'notify_port']
image_cache_path = os.path.join(index_dir, 'cache', 'downloaded')
rlist_cache_path = os.path.join(index_dir, 'cache', 'rlists')
# prepare config
log.info('[cpuvisor] Preparing config...')
def prepare_config():
template_config = os.path.join(templates_dir, 'config.prototxt')
output_config = os.path.join(component_paths['cpuvisor-srv'],
'config.%s.prototxt' % collection['name'])
# if the config file already exists, read in fields added by index_data first
restore_fields = False
if os.path.exists(output_config):
restore_fields = True
def get_field(field_name):
return cpuvisortls.get_config_field(
component_paths['cpuvisor-srv'], field_name, output_config)
dataset_im_paths = get_field('preproc_config.dataset_im_paths')
dataset_im_base_path = get_field(
'preproc_config.dataset_im_base_path')
dataset_feats_file = get_field('preproc_config.dataset_feats_file')
# write the new config file
replace_patterns = {
'<MODELS_PATH>': models_path,
'<NEG_IM_PATH>': negimgs_path,
'<NEG_IM_INDEX>': negidx_path,
'<NEG_FEATS_FILE>': negfeats_path,
'<SERVER_ENDPOINT>': server_endpoint,
'<NOTIFY_ENDPOINT>': notify_endpoint,
'<IMAGE_CACHE_PATH>': image_cache_path,
'<RLIST_CACHE_PATH>': rlist_cache_path
}
replace_patterns = list(replace_patterns.iteritems())
with open(template_config, 'r') as src_f:
with open(output_config, 'w') as dst_f:
utils.copy_replace(src_f, dst_f, replace_patterns)
# now restore fields added by index_data if required
if restore_fields:
def set_field(field_name, field_value):
cpuvisortls.set_config_field(component_paths['cpuvisor-srv'],
field_name, field_value,
output_config)
set_field('preproc_config.dataset_im_paths', dataset_im_paths)
set_field('preproc_config.dataset_im_base_path',
dataset_im_base_path)
set_field('preproc_config.dataset_feats_file', dataset_feats_file)
prepare_config()
# prepare start script
log.info('[cpuvisor] Preparing start script...')
def write_start_script():
outf = os.path.join(path, 'start.sh')
utils.write_template(templates_dir, 'start.sh', outf,
{'name': component_cfgs['collection']['name']})
os.chmod(outf, 0755)
write_start_script()
# download models
log.info('[cpuvisor] Getting models...')
cpuvisortls.download_models(models_path)
# download features for negative images
if os.path.exists(negfeats_path):
log.info('[cpuvisor] Negative feature file exists')
else:
log.info(
'[cpuvisor] Attempting to download features for negative images...'
)
if not cpuvisortls.download_neg_feats(negfeats_path):
# if no features could be downloaded, compute features using negative images instead
log.info(
'[cpuvisor] Could not download negative features - downloading negative training images instead...'
)
if not utils.touch_dir(negimgs_path, 'negimgs'):
cpuvisortls.download_neg_images(negimgs_path)
log.info('[cpuvisor] Computing features for negative images...')
with utils.change_cwd(
os.path.join(component_paths['cpuvisor-srv'], 'bin')):
utils.subproc_call_check([
'./cpuvisor_preproc', '--config_path',
'../config.%s.prototxt' %
component_cfgs['collection']['name'], '--nodsetfeats'
])
def index_cpuvisor(base_path, component_cfgs):
component_paths = component_cfgs['components']
links = component_cfgs['links']
collection = component_cfgs['collection']
index_dir = os.path.join(collection['paths']['index_data'], 'cpuvisor-srv')
precomputed_urls = {
'cAXESOpenMini': {
'paths': 'http://www.robots.ox.ac.uk/~vgg/software/deep_eval/releases/dsetpaths_cAXESOpen.txt',
'feats': 'http://www.robots.ox.ac.uk/~vgg/software/deep_eval/releases/dsetfeats_cAXESOpen_VGG_CNN_M_128.tgz'
},
'cAXESOpen': {
'paths': 'http://axis.ewi.utwente.nl/collections/cAXESOpen/dsetpaths_cAXESOpen.txt',
'feats': 'http://axis.ewi.utwente.nl/collections/cAXESOpen/dsetfeats_cAXESOpen_VGG_CNN_M_128.tgz'
},
'cAXESOpenCnn': {
'paths': 'http://axis.ewi.utwente.nl/collections/cAXESOpen/dsetpaths_cAXESOpenCnn.txt',
'feats': 'http://axis.ewi.utwente.nl/collections/cAXESOpen/dsetfeats_cAXESOpenCnn_VGG_CNN_M_128.tgz'
}
}
# ensure all precomputed urls exist
for dataset in precomputed_urls.keys():
urls = precomputed_urls[dataset]
exists = (utils.check_url_exists(urls['paths']) and
utils.check_url_exists(urls['feats']))
if not exists:
log.warning('[cpuvisor] Precomputed filelist/feature URLs for '
'dataset %s did not exist. Will compute features '
'for this dataset from scratch if the feature file '
'does not already exist.' % dataset)
del precomputed_urls[dataset]
for i in reversed(range(1,6)):
sys.stdout.write('Continuing in %d seconds...\r' % i)
sys.stdout.flush()
time.sleep(1)
sys.stdout.write('\033[K\r')
sys.stdout.flush()
# ensure index directory exists
try:
os.makedirs(index_dir)
except os.error:
pass
cpuvisortls = utils.import_python_module_from_path(component_paths['cpuvisor-srv'],
'download_data')
cpuvisorutil = utils.import_python_module_from_path(os.path.join(component_paths['cpuvisor-srv'], 'utils'),
'generate_imagelist')
# prepare paths
log.info('[cpuvisor] Determining dataset paths...')
dataset_keyframes_path = os.path.join(collection['paths']['private_data'], 'keyframes')
dataset_im_paths_file = os.path.join(index_dir, 'dsetpaths_%s.txt' % collection['name'])
dataset_feats_file = os.path.join(index_dir, 'dsetfeats_%s.binaryproto' % collection['name'])
# generate/download filelist for dataset
def download_filelist(collection_name, fname, root_dir):
if collection_name not in precomputed_urls:
return False
else:
url = precomputed_urls[collection_name]['paths']
log.info('[cpuvisor] Downloading dataset filelist...')
log.info('[cpuvisor] URL is: %s' % url)
assert(os.path.splitext(urlparse.urlparse(url).path)[1] == os.path.splitext(fname)[1])
utils.subproc_call_check(['wget -O %s %s' % (fname, url)], shell=True)
# re-generate and check for consistency
with utils.make_temp_directory() as temp_dir:
regen_fname = os.path.join(temp_dir, os.path.split(fname)[1])
generate_filelist(regen_fname, root_dir, True)
if filecmp.cmp(fname, regen_fname):
raise RuntimeError('Downloaded filelist for dataset <%s>: %s '
'is inconsistent with images found in '
'dataset directory: %s' %
(collection_name, fname, root_dir))
return True
def generate_filelist(fname, root_dir, silent=False):
if not silent:
log.info('[cpuvisor] Generating dataset filelist...')
cpuvisorutil.generate_imagelist(fname, root_dir)
if not os.path.exists(dataset_im_paths_file):
if not download_filelist(collection['name'], dataset_im_paths_file,
dataset_keyframes_path):
generate_filelist(dataset_im_paths_file, dataset_keyframes_path)
# update config with paths
log.info('[cpuvisor] Updating config with dataset paths...')
config_file_path = os.path.join(component_paths['cpuvisor-srv'],
'config.%s.prototxt' % collection['name'])
cpuvisortls.set_config_field(component_paths['cpuvisor-srv'],
'preproc_config.dataset_im_paths',
dataset_im_paths_file,
config_file_path)
cpuvisortls.set_config_field(component_paths['cpuvisor-srv'],
'preproc_config.dataset_im_base_path',
dataset_keyframes_path,
config_file_path)
cpuvisortls.set_config_field(component_paths['cpuvisor-srv'],
'preproc_config.dataset_feats_file',
dataset_feats_file,
config_file_path)
# compute/download features for dataset
def download_feats(collection_name, fname):
if collection_name not in precomputed_urls:
return False
else:
url = precomputed_urls[collection_name]['feats']
log.info('[cpuvisor] Downloading features for dataset...')
log.info('[cpuvisor] URL is: %s' % url)
(target_path, target_fname) = os.path.split(fname)
with utils.make_temp_directory() as temp_dir:
tarball_file_ext = os.path.splitext(urlparse.urlparse(url).path)[1]
tarball_fname = os.path.join(temp_dir, 'feats' + tarball_file_ext)
utils.subproc_call_check(['wget -O %s %s' % (tarball_fname, url)], shell=True)
with tarfile.open(tarball_fname) as tar:
tar_ifos = {x.name: x for x in tar.getmembers()}
if target_fname not in tar_ifos:
raise RuntimeError('Precomputed feature tarball does not contain required file')
else:
tar.extractall(target_path, [tar_ifos[target_fname]])
return True
def compute_feats(base_dir, collection_name):
log.info('[cpuvisor] Computing features for dataset...')
with utils.change_cwd(os.path.join(base_dir, 'bin')):
utils.subproc_call_check([
'./cpuvisor_preproc',
'--config_path', '../config.%s.prototxt' % collection_name,
'--nonegfeats'
])
if not os.path.exists(dataset_feats_file):
if not download_feats(collection['name'], dataset_feats_file):
compute_feats(component_paths['cpuvisor-srv'],
component_cfgs['collection']['name'])
def prepare_cpuvisor(base_path, component_cfgs):
component_paths = component_cfgs['components']
links = component_cfgs['links']
collection = component_cfgs['collection']
index_dir = os.path.join(collection['paths']['index_data'], 'cpuvisor-srv')
templates_dir = os.path.join('templates', 'cpuvisor-srv')
path = component_paths['cpuvisor-srv']
cpuvisortls = utils.import_python_module_from_path(component_paths['cpuvisor-srv'],
'download_data')
if not os.path.exists(os.path.join(component_paths['cpuvisor-srv'], 'bin')):
raise RuntimeError('[cpuvisor] Missing bin directory - compile before running link_components!')
# prepare endpoints and paths
models_path = os.path.join(component_paths['cpuvisor-srv'], 'model_data')
negimgs_path = os.path.join(component_paths['cpuvisor-srv'],
'server_data', 'neg_images')
negidx_path = os.path.join(component_paths['cpuvisor-srv'],
'negpaths.txt')
negfeats_path = os.path.join(component_paths['cpuvisor-srv'],
'server_data', 'negfeats.binaryproto')
server_endpoint = 'tcp://127.0.0.1:%d' % links['cpuvisor-srv']['server_port']
notify_endpoint = 'tcp://127.0.0.1:%d' % links['cpuvisor-srv']['notify_port']
image_cache_path = os.path.join(index_dir, 'cache', 'downloaded')
rlist_cache_path = os.path.join(index_dir, 'cache', 'rlists')
# prepare config
log.info('[cpuvisor] Preparing config...')
def prepare_config():
template_config = os.path.join(templates_dir, 'config.prototxt')
output_config = os.path.join(component_paths['cpuvisor-srv'],
'config.%s.prototxt' % collection['name'])
# if the config file already exists, read in fields added by index_data first
restore_fields = False
if os.path.exists(output_config):
restore_fields = True
def get_field(field_name):
return cpuvisortls.get_config_field(component_paths['cpuvisor-srv'],
field_name,
output_config)
dataset_im_paths = get_field('preproc_config.dataset_im_paths')
dataset_im_base_path = get_field('preproc_config.dataset_im_base_path')
dataset_feats_file = get_field('preproc_config.dataset_feats_file')
# write the new config file
replace_patterns = {
'<MODELS_PATH>': models_path,
'<NEG_IM_PATH>': negimgs_path,
'<NEG_IM_INDEX>': negidx_path,
'<NEG_FEATS_FILE>': negfeats_path,
'<SERVER_ENDPOINT>': server_endpoint,
'<NOTIFY_ENDPOINT>': notify_endpoint,
'<IMAGE_CACHE_PATH>': image_cache_path,
'<RLIST_CACHE_PATH>': rlist_cache_path
}
replace_patterns = list(replace_patterns.iteritems())
with open(template_config, 'r') as src_f:
with open(output_config, 'w') as dst_f:
utils.copy_replace(src_f, dst_f, replace_patterns)
# now restore fields added by index_data if required
if restore_fields:
def set_field(field_name, field_value):
cpuvisortls.set_config_field(component_paths['cpuvisor-srv'],
field_name,
field_value,
output_config)
set_field('preproc_config.dataset_im_paths', dataset_im_paths)
set_field('preproc_config.dataset_im_base_path', dataset_im_base_path)
set_field('preproc_config.dataset_feats_file', dataset_feats_file)
prepare_config()
# prepare start script
log.info('[cpuvisor] Preparing start script...')
def write_start_script():
outf = os.path.join(path, 'start.sh')
utils.write_template(templates_dir, 'start.sh', outf,
{'name': component_cfgs['collection']['name']})
os.chmod(outf, 0755)
write_start_script()
# download models
log.info('[cpuvisor] Getting models...')
cpuvisortls.download_models(models_path)
# download features for negative images
if os.path.exists(negfeats_path):
log.info('[cpuvisor] Negative feature file exists')
else:
log.info('[cpuvisor] Attempting to download features for negative images...')
if not cpuvisortls.download_neg_feats(negfeats_path):
# if no features could be downloaded, compute features using negative images instead
log.info('[cpuvisor] Could not download negative features - downloading negative training images instead...')
if not utils.touch_dir(negimgs_path, 'negimgs'):
cpuvisortls.download_neg_images(negimgs_path)
log.info('[cpuvisor] Computing features for negative images...')
with utils.change_cwd(os.path.join(component_paths['cpuvisor-srv'], 'bin')):
utils.subproc_call_check([
'./cpuvisor_preproc',
'--config_path', '../config.%s.prototxt' % component_cfgs['collection']['name'],
'--nodsetfeats'
])
