def collect_static_files(): activate_cmd = ". ./venv/bin/activate" collect_cmd = "echo yes | python manage.py collectstatic" with utils.change_cwd(path): cmd = collect_cmd if os.path.isdir("venv"): cmd = activate_cmd + " && " + collect_cmd os.system(cmd)
def index_limas(base_path, component_cfgs): components = component_cfgs['components'] links = component_cfgs['links'] data = component_cfgs['collection']['paths'] collection = component_cfgs['collection']['name'] conf_fn = os.path.join(components['limas'], 'conf', collection + '.py') os.environ['PATH'] = os.environ['PATH'] + ":" + os.path.join( components['limas'], 'bin') with utils.change_cwd(components['limas']): # index main video files cmd = ["scripts/shotdetection/index_videos.py", conf_fn, collection, os.path.join(data['private_data'], 'ffprobe')] utils.subproc_call_check(cmd) # index video-level metatada cmd = ["scripts/integration/index_meta.py", conf_fn, collection, os.path.join(data['private_data'], 'metadata')] utils.subproc_call_check(cmd) # index shot and keyframe data cmd = ["scripts/integration/index_shots_from_timings.py", conf_fn, collection, os.path.join(data['private_data'], 'shottimings')] utils.subproc_call_check(cmd) # index asr data cmd = ["scripts/integration/index_asr_from_timings.py", conf_fn, collection, os.path.join(data['private_data'], 'asr')] utils.subproc_call_check(cmd) # normalize data cmd = ["bin/limas", 'normalize', conf_fn ] utils.subproc_call_check(cmd) # index data to inverted files cmd = ["bin/limas", 'indexASR', conf_fn] utils.subproc_call_check(cmd) # create collection statistics cmd = ["bin/limas", 'es', conf_fn] utils.subproc_call_check(cmd)
def compute_feats(base_dir, collection_name): log.info('[cpuvisor] Computing features for dataset...') with utils.change_cwd(os.path.join(base_dir, 'bin')): utils.subproc_call_check([ './cpuvisor_preproc', '--config_path', '../config.%s.prototxt' % collection_name, '--nonegfeats' ])
def prepare_limas(base_path, component_cfgs): component_paths = component_cfgs['components'] links = component_cfgs['links'] collection = component_cfgs['collection'] templates_dir = os.path.join('templates', 'limas') path = component_paths['limas'] # prepare config log.info('[limas] Preparing config...') with utils.change_cwd(component_paths['limas']): set_env_replace_patterns = [ ('<directory to limas>', component_paths['limas']), ('<name of the collection that this instance should host>', collection['name']), ('<port under which the instance should run>', str(links['limas']['server_port'])), ('<directory to private data>', collection['paths']['private_data']) ] with open('set_env.sh.template', 'r') as src_f: with open('set_env.sh', 'w') as dst_f: utils.copy_replace(src_f, dst_f, set_env_replace_patterns) conf_replace_patterns = { '<COLLECTION_NAME>': collection['name'], '<PATH_TO_WEB_ACCESSIBLE_IMAGES>': collection['paths']['public_data'], '<PATH_TO_INDEX_STRUCTURES>': os.path.join(collection['paths']['index_data'], 'limas'), '<CPUVISOR_PORT>': str(links['cpuvisor-srv']['server_port']), '<CPUVISOR_NOTIFY_PORT>': str(links['cpuvisor-srv']['notify_port']), '<URL_TO_COLLECTION_PATH>': collection['url'], '<MONGODB_PORT>': str(links['mongodb']['server_port']), '<IMSEARCH_PORT>': str(links['imsearch-tools']['server_port']) } conf_replace_patterns = list(conf_replace_patterns.iteritems()) with open(os.path.join('conf', 'conf-template.py'), 'r') as src_f: with open(os.path.join('conf', collection['name'] + '.py'), 'w') as dst_f: utils.copy_replace(src_f, dst_f, conf_replace_patterns) # prepare start script log.info('[limas] Preparing start script...') def write_start_script(): outf = os.path.join(path, 'start.sh') utils.write_template(templates_dir, 'start.sh', outf, {'server_port': links['limas']['server_port'], 'base_dir': path, 'name': component_cfgs['collection']['name']}) os.chmod(outf, 0755) write_start_script()
def repo_clone(cmd, component, url, branch=None): default_val = '<PATH>' path = component_opts['components'][component] if path is None or path == default_val: return False if os.path.isdir(path): return False clone_cmd = ' '.join([cmd, 'clone', url, path]) subprocess.call(clone_cmd, shell=True) if branch: with utils.change_cwd(path): checkout_cmd = ' '.join([cmd, 'checkout', branch]) subprocess.call(checkout_cmd, shell=True) return True
def prepare_limas(base_path, component_cfgs): component_paths = component_cfgs['components'] links = component_cfgs['links'] collection = component_cfgs['collection'] templates_dir = os.path.join('templates', 'limas') path = component_paths['limas'] # prepare config log.info('[limas] Preparing config...') with utils.change_cwd(component_paths['limas']): set_env_replace_patterns = [ ('<directory to limas>', component_paths['limas']), ('<name of the collection that this instance should host>', collection['name']), ('<port under which the instance should run>', str(links['limas']['server_port'])), ('<directory to private data>', collection['paths']['private_data']) ] with open('set_env.sh.template', 'r') as src_f: with open('set_env.sh', 'w') as dst_f: utils.copy_replace(src_f, dst_f, set_env_replace_patterns) conf_replace_patterns = { '<COLLECTION_NAME>': collection['name'], '<PATH_TO_WEB_ACCESSIBLE_IMAGES>': collection['paths']['public_data'], '<PATH_TO_INDEX_STRUCTURES>': os.path.join(collection['paths']['index_data'], 'limas'), '<CPUVISOR_PORT>': str(links['cpuvisor-srv']['server_port']), '<CPUVISOR_NOTIFY_PORT>': str(links['cpuvisor-srv']['notify_port']), '<URL_TO_COLLECTION_PATH>': collection['url'], '<MONGODB_PORT>': str(links['mongodb']['server_port']), '<IMSEARCH_PORT>': str(links['imsearch-tools']['server_port']) } conf_replace_patterns = list(conf_replace_patterns.iteritems()) with open(os.path.join('conf', 'conf-template.py'), 'r') as src_f: with open(os.path.join('conf', collection['name'] + '.py'), 'w') as dst_f: utils.copy_replace(src_f, dst_f, conf_replace_patterns) # prepare start script log.info('[limas] Preparing start script...') def write_start_script(): outf = os.path.join(path, 'start.sh') utils.write_template( templates_dir, 'start.sh', outf, { 'server_port': links['limas']['server_port'], 'base_dir': path, 'name': component_cfgs['collection']['name'] }) os.chmod(outf, 0755) write_start_script()
def prepare_cpuvisor(base_path, component_cfgs): component_paths = component_cfgs['components'] links = component_cfgs['links'] collection = component_cfgs['collection'] index_dir = os.path.join(collection['paths']['index_data'], 'cpuvisor-srv') templates_dir = os.path.join('templates', 'cpuvisor-srv') path = component_paths['cpuvisor-srv'] cpuvisortls = utils.import_python_module_from_path( component_paths['cpuvisor-srv'], 'download_data') if not os.path.exists(os.path.join(component_paths['cpuvisor-srv'], 'bin')): raise RuntimeError( '[cpuvisor] Missing bin directory - compile before running link_components!' ) # prepare endpoints and paths models_path = os.path.join(component_paths['cpuvisor-srv'], 'model_data') negimgs_path = os.path.join(component_paths['cpuvisor-srv'], 'server_data', 'neg_images') negidx_path = os.path.join(component_paths['cpuvisor-srv'], 'negpaths.txt') negfeats_path = os.path.join(component_paths['cpuvisor-srv'], 'server_data', 'negfeats.binaryproto') server_endpoint = 'tcp://127.0.0.1:%d' % links['cpuvisor-srv'][ 'server_port'] notify_endpoint = 'tcp://127.0.0.1:%d' % links['cpuvisor-srv'][ 'notify_port'] image_cache_path = os.path.join(index_dir, 'cache', 'downloaded') rlist_cache_path = os.path.join(index_dir, 'cache', 'rlists') # prepare config log.info('[cpuvisor] Preparing config...') def prepare_config(): template_config = os.path.join(templates_dir, 'config.prototxt') output_config = os.path.join(component_paths['cpuvisor-srv'], 'config.%s.prototxt' % collection['name']) # if the config file already exists, read in fields added by index_data first restore_fields = False if os.path.exists(output_config): restore_fields = True def get_field(field_name): return cpuvisortls.get_config_field( component_paths['cpuvisor-srv'], field_name, output_config) dataset_im_paths = get_field('preproc_config.dataset_im_paths') dataset_im_base_path = get_field( 'preproc_config.dataset_im_base_path') dataset_feats_file = get_field('preproc_config.dataset_feats_file') # write the new config file replace_patterns = { '<MODELS_PATH>': models_path, '<NEG_IM_PATH>': negimgs_path, '<NEG_IM_INDEX>': negidx_path, '<NEG_FEATS_FILE>': negfeats_path, '<SERVER_ENDPOINT>': server_endpoint, '<NOTIFY_ENDPOINT>': notify_endpoint, '<IMAGE_CACHE_PATH>': image_cache_path, '<RLIST_CACHE_PATH>': rlist_cache_path } replace_patterns = list(replace_patterns.iteritems()) with open(template_config, 'r') as src_f: with open(output_config, 'w') as dst_f: utils.copy_replace(src_f, dst_f, replace_patterns) # now restore fields added by index_data if required if restore_fields: def set_field(field_name, field_value): cpuvisortls.set_config_field(component_paths['cpuvisor-srv'], field_name, field_value, output_config) set_field('preproc_config.dataset_im_paths', dataset_im_paths) set_field('preproc_config.dataset_im_base_path', dataset_im_base_path) set_field('preproc_config.dataset_feats_file', dataset_feats_file) prepare_config() # prepare start script log.info('[cpuvisor] Preparing start script...') def write_start_script(): outf = os.path.join(path, 'start.sh') utils.write_template(templates_dir, 'start.sh', outf, {'name': component_cfgs['collection']['name']}) os.chmod(outf, 0755) write_start_script() # download models log.info('[cpuvisor] Getting models...') cpuvisortls.download_models(models_path) # download features for negative images if os.path.exists(negfeats_path): log.info('[cpuvisor] Negative feature file exists') else: log.info( '[cpuvisor] Attempting to download features for negative images...' ) if not cpuvisortls.download_neg_feats(negfeats_path): # if no features could be downloaded, compute features using negative images instead log.info( '[cpuvisor] Could not download negative features - downloading negative training images instead...' ) if not utils.touch_dir(negimgs_path, 'negimgs'): cpuvisortls.download_neg_images(negimgs_path) log.info('[cpuvisor] Computing features for negative images...') with utils.change_cwd( os.path.join(component_paths['cpuvisor-srv'], 'bin')): utils.subproc_call_check([ './cpuvisor_preproc', '--config_path', '../config.%s.prototxt' % component_cfgs['collection']['name'], '--nodsetfeats' ])
def prepare_cpuvisor(base_path, component_cfgs): component_paths = component_cfgs['components'] links = component_cfgs['links'] collection = component_cfgs['collection'] index_dir = os.path.join(collection['paths']['index_data'], 'cpuvisor-srv') templates_dir = os.path.join('templates', 'cpuvisor-srv') path = component_paths['cpuvisor-srv'] cpuvisortls = utils.import_python_module_from_path(component_paths['cpuvisor-srv'], 'download_data') if not os.path.exists(os.path.join(component_paths['cpuvisor-srv'], 'bin')): raise RuntimeError('[cpuvisor] Missing bin directory - compile before running link_components!') # prepare endpoints and paths models_path = os.path.join(component_paths['cpuvisor-srv'], 'model_data') negimgs_path = os.path.join(component_paths['cpuvisor-srv'], 'server_data', 'neg_images') negidx_path = os.path.join(component_paths['cpuvisor-srv'], 'negpaths.txt') negfeats_path = os.path.join(component_paths['cpuvisor-srv'], 'server_data', 'negfeats.binaryproto') server_endpoint = 'tcp://127.0.0.1:%d' % links['cpuvisor-srv']['server_port'] notify_endpoint = 'tcp://127.0.0.1:%d' % links['cpuvisor-srv']['notify_port'] image_cache_path = os.path.join(index_dir, 'cache', 'downloaded') rlist_cache_path = os.path.join(index_dir, 'cache', 'rlists') # prepare config log.info('[cpuvisor] Preparing config...') def prepare_config(): template_config = os.path.join(templates_dir, 'config.prototxt') output_config = os.path.join(component_paths['cpuvisor-srv'], 'config.%s.prototxt' % collection['name']) # if the config file already exists, read in fields added by index_data first restore_fields = False if os.path.exists(output_config): restore_fields = True def get_field(field_name): return cpuvisortls.get_config_field(component_paths['cpuvisor-srv'], field_name, output_config) dataset_im_paths = get_field('preproc_config.dataset_im_paths') dataset_im_base_path = get_field('preproc_config.dataset_im_base_path') dataset_feats_file = get_field('preproc_config.dataset_feats_file') # write the new config file replace_patterns = { '<MODELS_PATH>': models_path, '<NEG_IM_PATH>': negimgs_path, '<NEG_IM_INDEX>': negidx_path, '<NEG_FEATS_FILE>': negfeats_path, '<SERVER_ENDPOINT>': server_endpoint, '<NOTIFY_ENDPOINT>': notify_endpoint, '<IMAGE_CACHE_PATH>': image_cache_path, '<RLIST_CACHE_PATH>': rlist_cache_path } replace_patterns = list(replace_patterns.iteritems()) with open(template_config, 'r') as src_f: with open(output_config, 'w') as dst_f: utils.copy_replace(src_f, dst_f, replace_patterns) # now restore fields added by index_data if required if restore_fields: def set_field(field_name, field_value): cpuvisortls.set_config_field(component_paths['cpuvisor-srv'], field_name, field_value, output_config) set_field('preproc_config.dataset_im_paths', dataset_im_paths) set_field('preproc_config.dataset_im_base_path', dataset_im_base_path) set_field('preproc_config.dataset_feats_file', dataset_feats_file) prepare_config() # prepare start script log.info('[cpuvisor] Preparing start script...') def write_start_script(): outf = os.path.join(path, 'start.sh') utils.write_template(templates_dir, 'start.sh', outf, {'name': component_cfgs['collection']['name']}) os.chmod(outf, 0755) write_start_script() # download models log.info('[cpuvisor] Getting models...') cpuvisortls.download_models(models_path) # download features for negative images if os.path.exists(negfeats_path): log.info('[cpuvisor] Negative feature file exists') else: log.info('[cpuvisor] Attempting to download features for negative images...') if not cpuvisortls.download_neg_feats(negfeats_path): # if no features could be downloaded, compute features using negative images instead log.info('[cpuvisor] Could not download negative features - downloading negative training images instead...') if not utils.touch_dir(negimgs_path, 'negimgs'): cpuvisortls.download_neg_images(negimgs_path) log.info('[cpuvisor] Computing features for negative images...') with utils.change_cwd(os.path.join(component_paths['cpuvisor-srv'], 'bin')): utils.subproc_call_check([ './cpuvisor_preproc', '--config_path', '../config.%s.prototxt' % component_cfgs['collection']['name'], '--nodsetfeats' ])