def collect_static_files():
activate_cmd = ". ./venv/bin/activate"
collect_cmd = "echo yes | python manage.py collectstatic"
with utils.change_cwd(path):
cmd = collect_cmd
if os.path.isdir("venv"):
cmd = activate_cmd + " && " + collect_cmd
os.system(cmd)
def collect_static_files():
activate_cmd = ". ./venv/bin/activate"
collect_cmd = "echo yes | python manage.py collectstatic"
with utils.change_cwd(path):
cmd = collect_cmd
if os.path.isdir("venv"):
cmd = activate_cmd + " && " + collect_cmd
os.system(cmd)
def index_limas(base_path, component_cfgs):
components = component_cfgs['components']
links = component_cfgs['links']
data = component_cfgs['collection']['paths']
collection = component_cfgs['collection']['name']
conf_fn = os.path.join(components['limas'], 'conf', collection + '.py')
os.environ['PATH'] = os.environ['PATH'] + ":" + os.path.join( components['limas'], 'bin')
with utils.change_cwd(components['limas']):
# index main video files
cmd = ["scripts/shotdetection/index_videos.py",
conf_fn,
collection,
os.path.join(data['private_data'], 'ffprobe')]
utils.subproc_call_check(cmd)
# index video-level metatada
cmd = ["scripts/integration/index_meta.py",
conf_fn,
collection,
os.path.join(data['private_data'], 'metadata')]
utils.subproc_call_check(cmd)
# index shot and keyframe data
cmd = ["scripts/integration/index_shots_from_timings.py",
conf_fn,
collection,
os.path.join(data['private_data'], 'shottimings')]
utils.subproc_call_check(cmd)
# index asr data
cmd = ["scripts/integration/index_asr_from_timings.py",
conf_fn,
collection,
os.path.join(data['private_data'], 'asr')]
utils.subproc_call_check(cmd)
# normalize data
cmd = ["bin/limas",
'normalize',
conf_fn ]
utils.subproc_call_check(cmd)
# index data to inverted files
cmd = ["bin/limas",
'indexASR',
conf_fn]
utils.subproc_call_check(cmd)
# create collection statistics
cmd = ["bin/limas",
'es',
conf_fn]
utils.subproc_call_check(cmd)
def compute_feats(base_dir, collection_name):
log.info('[cpuvisor] Computing features for dataset...')
with utils.change_cwd(os.path.join(base_dir, 'bin')):
utils.subproc_call_check([
'./cpuvisor_preproc',
'--config_path', '../config.%s.prototxt' % collection_name,
'--nonegfeats'
])
def prepare_limas(base_path, component_cfgs):
component_paths = component_cfgs['components']
links = component_cfgs['links']
collection = component_cfgs['collection']
templates_dir = os.path.join('templates', 'limas')
path = component_paths['limas']
# prepare config
log.info('[limas] Preparing config...')
with utils.change_cwd(component_paths['limas']):
set_env_replace_patterns = [
('<directory to limas>',
component_paths['limas']),
('<name of the collection that this instance should host>',
collection['name']),
('<port under which the instance should run>',
str(links['limas']['server_port'])),
('<directory to private data>',
collection['paths']['private_data'])
]
with open('set_env.sh.template', 'r') as src_f:
with open('set_env.sh', 'w') as dst_f:
utils.copy_replace(src_f, dst_f, set_env_replace_patterns)
conf_replace_patterns = {
'<COLLECTION_NAME>': collection['name'],
'<PATH_TO_WEB_ACCESSIBLE_IMAGES>': collection['paths']['public_data'],
'<PATH_TO_INDEX_STRUCTURES>': os.path.join(collection['paths']['index_data'],
'limas'),
'<CPUVISOR_PORT>': str(links['cpuvisor-srv']['server_port']),
'<CPUVISOR_NOTIFY_PORT>': str(links['cpuvisor-srv']['notify_port']),
'<URL_TO_COLLECTION_PATH>': collection['url'],
'<MONGODB_PORT>': str(links['mongodb']['server_port']),
'<IMSEARCH_PORT>': str(links['imsearch-tools']['server_port'])
}
conf_replace_patterns = list(conf_replace_patterns.iteritems())
with open(os.path.join('conf', 'conf-template.py'), 'r') as src_f:
with open(os.path.join('conf', collection['name'] + '.py'), 'w') as dst_f:
utils.copy_replace(src_f, dst_f, conf_replace_patterns)
# prepare start script
log.info('[limas] Preparing start script...')
def write_start_script():
outf = os.path.join(path, 'start.sh')
utils.write_template(templates_dir, 'start.sh', outf,
{'server_port': links['limas']['server_port'],
'base_dir': path,
'name': component_cfgs['collection']['name']})
os.chmod(outf, 0755)
write_start_script()
def repo_clone(cmd, component, url, branch=None):
default_val = '<PATH>'
path = component_opts['components'][component]
if path is None or path == default_val:
return False
if os.path.isdir(path):
return False
clone_cmd = ' '.join([cmd, 'clone', url, path])
subprocess.call(clone_cmd, shell=True)
if branch:
with utils.change_cwd(path):
checkout_cmd = ' '.join([cmd, 'checkout', branch])
subprocess.call(checkout_cmd, shell=True)
return True
def repo_clone(cmd, component, url, branch=None):
default_val = '<PATH>'
path = component_opts['components'][component]
if path is None or path == default_val:
return False
if os.path.isdir(path):
return False
clone_cmd = ' '.join([cmd, 'clone', url, path])
subprocess.call(clone_cmd, shell=True)
if branch:
with utils.change_cwd(path):
checkout_cmd = ' '.join([cmd, 'checkout', branch])
subprocess.call(checkout_cmd, shell=True)
return True
def prepare_limas(base_path, component_cfgs):
component_paths = component_cfgs['components']
links = component_cfgs['links']
collection = component_cfgs['collection']
templates_dir = os.path.join('templates', 'limas')
path = component_paths['limas']
# prepare config
log.info('[limas] Preparing config...')
with utils.change_cwd(component_paths['limas']):
set_env_replace_patterns = [
('<directory to limas>', component_paths['limas']),
('<name of the collection that this instance should host>',
collection['name']),
('<port under which the instance should run>',
str(links['limas']['server_port'])),
('<directory to private data>',
collection['paths']['private_data'])
]
with open('set_env.sh.template', 'r') as src_f:
with open('set_env.sh', 'w') as dst_f:
utils.copy_replace(src_f, dst_f, set_env_replace_patterns)
conf_replace_patterns = {
'<COLLECTION_NAME>':
collection['name'],
'<PATH_TO_WEB_ACCESSIBLE_IMAGES>':
collection['paths']['public_data'],
'<PATH_TO_INDEX_STRUCTURES>':
os.path.join(collection['paths']['index_data'], 'limas'),
'<CPUVISOR_PORT>':
str(links['cpuvisor-srv']['server_port']),
'<CPUVISOR_NOTIFY_PORT>':
str(links['cpuvisor-srv']['notify_port']),
'<URL_TO_COLLECTION_PATH>':
collection['url'],
'<MONGODB_PORT>':
str(links['mongodb']['server_port']),
'<IMSEARCH_PORT>':
str(links['imsearch-tools']['server_port'])
}
conf_replace_patterns = list(conf_replace_patterns.iteritems())
with open(os.path.join('conf', 'conf-template.py'), 'r') as src_f:
with open(os.path.join('conf', collection['name'] + '.py'),
'w') as dst_f:
utils.copy_replace(src_f, dst_f, conf_replace_patterns)
# prepare start script
log.info('[limas] Preparing start script...')
def write_start_script():
outf = os.path.join(path, 'start.sh')
utils.write_template(
templates_dir, 'start.sh', outf, {
'server_port': links['limas']['server_port'],
'base_dir': path,
'name': component_cfgs['collection']['name']
})
os.chmod(outf, 0755)
write_start_script()
def prepare_cpuvisor(base_path, component_cfgs):
component_paths = component_cfgs['components']
links = component_cfgs['links']
collection = component_cfgs['collection']
index_dir = os.path.join(collection['paths']['index_data'], 'cpuvisor-srv')
templates_dir = os.path.join('templates', 'cpuvisor-srv')
path = component_paths['cpuvisor-srv']
cpuvisortls = utils.import_python_module_from_path(
component_paths['cpuvisor-srv'], 'download_data')
if not os.path.exists(os.path.join(component_paths['cpuvisor-srv'],
'bin')):
raise RuntimeError(
'[cpuvisor] Missing bin directory - compile before running link_components!'
)
# prepare endpoints and paths
models_path = os.path.join(component_paths['cpuvisor-srv'], 'model_data')
negimgs_path = os.path.join(component_paths['cpuvisor-srv'], 'server_data',
'neg_images')
negidx_path = os.path.join(component_paths['cpuvisor-srv'], 'negpaths.txt')
negfeats_path = os.path.join(component_paths['cpuvisor-srv'],
'server_data', 'negfeats.binaryproto')
server_endpoint = 'tcp://127.0.0.1:%d' % links['cpuvisor-srv'][
'server_port']
notify_endpoint = 'tcp://127.0.0.1:%d' % links['cpuvisor-srv'][
'notify_port']
image_cache_path = os.path.join(index_dir, 'cache', 'downloaded')
rlist_cache_path = os.path.join(index_dir, 'cache', 'rlists')
# prepare config
log.info('[cpuvisor] Preparing config...')
def prepare_config():
template_config = os.path.join(templates_dir, 'config.prototxt')
output_config = os.path.join(component_paths['cpuvisor-srv'],
'config.%s.prototxt' % collection['name'])
# if the config file already exists, read in fields added by index_data first
restore_fields = False
if os.path.exists(output_config):
restore_fields = True
def get_field(field_name):
return cpuvisortls.get_config_field(
component_paths['cpuvisor-srv'], field_name, output_config)
dataset_im_paths = get_field('preproc_config.dataset_im_paths')
dataset_im_base_path = get_field(
'preproc_config.dataset_im_base_path')
dataset_feats_file = get_field('preproc_config.dataset_feats_file')
# write the new config file
replace_patterns = {
'<MODELS_PATH>': models_path,
'<NEG_IM_PATH>': negimgs_path,
'<NEG_IM_INDEX>': negidx_path,
'<NEG_FEATS_FILE>': negfeats_path,
'<SERVER_ENDPOINT>': server_endpoint,
'<NOTIFY_ENDPOINT>': notify_endpoint,
'<IMAGE_CACHE_PATH>': image_cache_path,
'<RLIST_CACHE_PATH>': rlist_cache_path
}
replace_patterns = list(replace_patterns.iteritems())
with open(template_config, 'r') as src_f:
with open(output_config, 'w') as dst_f:
utils.copy_replace(src_f, dst_f, replace_patterns)
# now restore fields added by index_data if required
if restore_fields:
def set_field(field_name, field_value):
cpuvisortls.set_config_field(component_paths['cpuvisor-srv'],
field_name, field_value,
output_config)
set_field('preproc_config.dataset_im_paths', dataset_im_paths)
set_field('preproc_config.dataset_im_base_path',
dataset_im_base_path)
set_field('preproc_config.dataset_feats_file', dataset_feats_file)
prepare_config()
# prepare start script
log.info('[cpuvisor] Preparing start script...')
def write_start_script():
outf = os.path.join(path, 'start.sh')
utils.write_template(templates_dir, 'start.sh', outf,
{'name': component_cfgs['collection']['name']})
os.chmod(outf, 0755)
write_start_script()
# download models
log.info('[cpuvisor] Getting models...')
cpuvisortls.download_models(models_path)
# download features for negative images
if os.path.exists(negfeats_path):
log.info('[cpuvisor] Negative feature file exists')
else:
log.info(
'[cpuvisor] Attempting to download features for negative images...'
)
if not cpuvisortls.download_neg_feats(negfeats_path):
# if no features could be downloaded, compute features using negative images instead
log.info(
'[cpuvisor] Could not download negative features - downloading negative training images instead...'
)
if not utils.touch_dir(negimgs_path, 'negimgs'):
cpuvisortls.download_neg_images(negimgs_path)
log.info('[cpuvisor] Computing features for negative images...')
with utils.change_cwd(
os.path.join(component_paths['cpuvisor-srv'], 'bin')):
utils.subproc_call_check([
'./cpuvisor_preproc', '--config_path',
'../config.%s.prototxt' %
component_cfgs['collection']['name'], '--nodsetfeats'
])
def prepare_cpuvisor(base_path, component_cfgs):
component_paths = component_cfgs['components']
links = component_cfgs['links']
collection = component_cfgs['collection']
index_dir = os.path.join(collection['paths']['index_data'], 'cpuvisor-srv')
templates_dir = os.path.join('templates', 'cpuvisor-srv')
path = component_paths['cpuvisor-srv']
cpuvisortls = utils.import_python_module_from_path(component_paths['cpuvisor-srv'],
'download_data')
if not os.path.exists(os.path.join(component_paths['cpuvisor-srv'], 'bin')):
raise RuntimeError('[cpuvisor] Missing bin directory - compile before running link_components!')
# prepare endpoints and paths
models_path = os.path.join(component_paths['cpuvisor-srv'], 'model_data')
negimgs_path = os.path.join(component_paths['cpuvisor-srv'],
'server_data', 'neg_images')
negidx_path = os.path.join(component_paths['cpuvisor-srv'],
'negpaths.txt')
negfeats_path = os.path.join(component_paths['cpuvisor-srv'],
'server_data', 'negfeats.binaryproto')
server_endpoint = 'tcp://127.0.0.1:%d' % links['cpuvisor-srv']['server_port']
notify_endpoint = 'tcp://127.0.0.1:%d' % links['cpuvisor-srv']['notify_port']
image_cache_path = os.path.join(index_dir, 'cache', 'downloaded')
rlist_cache_path = os.path.join(index_dir, 'cache', 'rlists')
# prepare config
log.info('[cpuvisor] Preparing config...')
def prepare_config():
template_config = os.path.join(templates_dir, 'config.prototxt')
output_config = os.path.join(component_paths['cpuvisor-srv'],
'config.%s.prototxt' % collection['name'])
# if the config file already exists, read in fields added by index_data first
restore_fields = False
if os.path.exists(output_config):
restore_fields = True
def get_field(field_name):
return cpuvisortls.get_config_field(component_paths['cpuvisor-srv'],
field_name,
output_config)
dataset_im_paths = get_field('preproc_config.dataset_im_paths')
dataset_im_base_path = get_field('preproc_config.dataset_im_base_path')
dataset_feats_file = get_field('preproc_config.dataset_feats_file')
# write the new config file
replace_patterns = {
'<MODELS_PATH>': models_path,
'<NEG_IM_PATH>': negimgs_path,
'<NEG_IM_INDEX>': negidx_path,
'<NEG_FEATS_FILE>': negfeats_path,
'<SERVER_ENDPOINT>': server_endpoint,
'<NOTIFY_ENDPOINT>': notify_endpoint,
'<IMAGE_CACHE_PATH>': image_cache_path,
'<RLIST_CACHE_PATH>': rlist_cache_path
}
replace_patterns = list(replace_patterns.iteritems())
with open(template_config, 'r') as src_f:
with open(output_config, 'w') as dst_f:
utils.copy_replace(src_f, dst_f, replace_patterns)
# now restore fields added by index_data if required
if restore_fields:
def set_field(field_name, field_value):
cpuvisortls.set_config_field(component_paths['cpuvisor-srv'],
field_name,
field_value,
output_config)
set_field('preproc_config.dataset_im_paths', dataset_im_paths)
set_field('preproc_config.dataset_im_base_path', dataset_im_base_path)
set_field('preproc_config.dataset_feats_file', dataset_feats_file)
prepare_config()
# prepare start script
log.info('[cpuvisor] Preparing start script...')
def write_start_script():
outf = os.path.join(path, 'start.sh')
utils.write_template(templates_dir, 'start.sh', outf,
{'name': component_cfgs['collection']['name']})
os.chmod(outf, 0755)
write_start_script()
# download models
log.info('[cpuvisor] Getting models...')
cpuvisortls.download_models(models_path)
# download features for negative images
if os.path.exists(negfeats_path):
log.info('[cpuvisor] Negative feature file exists')
else:
log.info('[cpuvisor] Attempting to download features for negative images...')
if not cpuvisortls.download_neg_feats(negfeats_path):
# if no features could be downloaded, compute features using negative images instead
log.info('[cpuvisor] Could not download negative features - downloading negative training images instead...')
if not utils.touch_dir(negimgs_path, 'negimgs'):
cpuvisortls.download_neg_images(negimgs_path)
log.info('[cpuvisor] Computing features for negative images...')
with utils.change_cwd(os.path.join(component_paths['cpuvisor-srv'], 'bin')):
utils.subproc_call_check([
'./cpuvisor_preproc',
'--config_path', '../config.%s.prototxt' % component_cfgs['collection']['name'],
'--nodsetfeats'
])