def prepare_nginx(base_path, component_cfgs):
links = component_cfgs['links']
collection = component_cfgs['collection']['name']
components = component_cfgs['components']
templates_dir = os.path.join('templates', 'nginx')
path = os.path.join(components['nginx'], 'conf')
# place nginx config in root directory if directory specified by nginx does not exist
if not os.path.isdir(path):
path = ''
# prepare config
log.info('Preparing nginx config...')
replace_patterns = {
'<AXES-HOME>': components['axes-home'],
'<AXES-RESEARCH>': components['axes-research'],
'<NGINX_PORT>': str(links['nginx']['server_port']),
'<HOME>': base_path,
'<PUBLIC_DATA>': component_cfgs['collection']['paths']['public_data']
}
replace_patterns = list(replace_patterns.iteritems())
with open(os.path.join(templates_dir, 'nginx.conf'), 'r') as src_f:
with open(os.path.join(path, 'nginx.conf'), 'w') as dst_f:
utils.copy_replace(src_f, dst_f, replace_patterns)
def prepare_supervisor(base_path, component_cfgs):
links = component_cfgs['links']
collection = component_cfgs['collection']['name']
components = component_cfgs['components']
templates_dir = os.path.join('templates', 'supervisor')
# prepare config
log.info('Preparing supervisor configuration...')
# TODO: add either axes-research or axes-home or both to default target
# depending on value of config['limas']['frontend']
set_env_replace_patterns = {
'<INDEX_PATH>': component_cfgs['collection']['paths']['index_data'],
'<LIMAS>': components['limas'],
'<CPUVISOR-SRV>': components['cpuvisor-srv'],
'<IMSEARCH-TOOLS>': components['imsearch-tools'],
'<AXES-HOME>': components['axes-home'],
'<AXES-RESEARCH>': components['axes-research'],
'<NGINX>': components['nginx'] or '',
'<MONGODB>': components['mongodb'],
'<MONGODB_PORT': str(links['mongodb']['server_port'])
}
set_env_replace_patterns = list(set_env_replace_patterns.iteritems())
with open(os.path.join(templates_dir, 'supervisord.conf.template'),
'r') as src_f:
with open('supervisord.conf', 'w') as dst_f:
utils.copy_replace(src_f, dst_f, set_env_replace_patterns)
def prepare_supervisor(base_path, component_cfgs):
links = component_cfgs['links']
collection = component_cfgs['collection']['name']
components = component_cfgs['components']
templates_dir = os.path.join('templates', 'supervisor')
# prepare config
log.info('Preparing supervisor configuration...')
# TODO: add either axes-research or axes-home or both to default target
# depending on value of config['limas']['frontend']
set_env_replace_patterns = {
'<INDEX_PATH>': component_cfgs['collection']['paths']['index_data'],
'<LIMAS>': components['limas'],
'<CPUVISOR-SRV>': components['cpuvisor-srv'],
'<IMSEARCH-TOOLS>': components['imsearch-tools'],
'<AXES-HOME>': components['axes-home'],
'<AXES-RESEARCH>': components['axes-research'],
'<NGINX>': components['nginx'] or '',
'<MONGODB>': components['mongodb'],
'<MONGODB_PORT': str(links['mongodb']['server_port'])
}
set_env_replace_patterns = list(set_env_replace_patterns.iteritems())
with open(os.path.join(templates_dir, 'supervisord.conf.template'), 'r') as src_f:
with open('supervisord.conf', 'w') as dst_f:
utils.copy_replace(src_f, dst_f, set_env_replace_patterns)
def prepare_nginx(base_path, component_cfgs):
links = component_cfgs['links']
collection = component_cfgs['collection']['name']
components = component_cfgs['components']
templates_dir = os.path.join('templates', 'nginx')
path = os.path.join(components['nginx'], 'conf')
# place nginx config in root directory if directory specified by nginx does not exist
if not os.path.isdir(path):
path = ''
# prepare config
log.info('Preparing nginx config...')
replace_patterns = {
'<AXES-HOME>': components['axes-home'],
'<AXES-RESEARCH>': components['axes-research'],
'<NGINX_PORT>': str(links['nginx']['server_port']),
'<HOME>': base_path,
'<PUBLIC_DATA>': component_cfgs['collection']['paths']['public_data']
}
replace_patterns = list(replace_patterns.iteritems())
with open(os.path.join(templates_dir, 'nginx.conf'), 'r') as src_f:
with open(os.path.join(path, 'nginx.conf'), 'w') as dst_f:
utils.copy_replace(src_f, dst_f, replace_patterns)
def prepare_limas(base_path, component_cfgs):
component_paths = component_cfgs['components']
links = component_cfgs['links']
collection = component_cfgs['collection']
templates_dir = os.path.join('templates', 'limas')
path = component_paths['limas']
# prepare config
log.info('[limas] Preparing config...')
with utils.change_cwd(component_paths['limas']):
set_env_replace_patterns = [
('<directory to limas>',
component_paths['limas']),
('<name of the collection that this instance should host>',
collection['name']),
('<port under which the instance should run>',
str(links['limas']['server_port'])),
('<directory to private data>',
collection['paths']['private_data'])
]
with open('set_env.sh.template', 'r') as src_f:
with open('set_env.sh', 'w') as dst_f:
utils.copy_replace(src_f, dst_f, set_env_replace_patterns)
conf_replace_patterns = {
'<COLLECTION_NAME>': collection['name'],
'<PATH_TO_WEB_ACCESSIBLE_IMAGES>': collection['paths']['public_data'],
'<PATH_TO_INDEX_STRUCTURES>': os.path.join(collection['paths']['index_data'],
'limas'),
'<CPUVISOR_PORT>': str(links['cpuvisor-srv']['server_port']),
'<CPUVISOR_NOTIFY_PORT>': str(links['cpuvisor-srv']['notify_port']),
'<URL_TO_COLLECTION_PATH>': collection['url'],
'<MONGODB_PORT>': str(links['mongodb']['server_port']),
'<IMSEARCH_PORT>': str(links['imsearch-tools']['server_port'])
}
conf_replace_patterns = list(conf_replace_patterns.iteritems())
with open(os.path.join('conf', 'conf-template.py'), 'r') as src_f:
with open(os.path.join('conf', collection['name'] + '.py'), 'w') as dst_f:
utils.copy_replace(src_f, dst_f, conf_replace_patterns)
# prepare start script
log.info('[limas] Preparing start script...')
def write_start_script():
outf = os.path.join(path, 'start.sh')
utils.write_template(templates_dir, 'start.sh', outf,
{'server_port': links['limas']['server_port'],
'base_dir': path,
'name': component_cfgs['collection']['name']})
os.chmod(outf, 0755)
write_start_script()
def prepare_config():
template_config = os.path.join(templates_dir, 'config.prototxt')
output_config = os.path.join(component_paths['cpuvisor-srv'],
'config.%s.prototxt' % collection['name'])
# if the config file already exists, read in fields added by index_data first
restore_fields = False
if os.path.exists(output_config):
restore_fields = True
def get_field(field_name):
return cpuvisortls.get_config_field(
component_paths['cpuvisor-srv'], field_name, output_config)
dataset_im_paths = get_field('preproc_config.dataset_im_paths')
dataset_im_base_path = get_field(
'preproc_config.dataset_im_base_path')
dataset_feats_file = get_field('preproc_config.dataset_feats_file')
# write the new config file
replace_patterns = {
'<MODELS_PATH>': models_path,
'<NEG_IM_PATH>': negimgs_path,
'<NEG_IM_INDEX>': negidx_path,
'<NEG_FEATS_FILE>': negfeats_path,
'<SERVER_ENDPOINT>': server_endpoint,
'<NOTIFY_ENDPOINT>': notify_endpoint,
'<IMAGE_CACHE_PATH>': image_cache_path,
'<RLIST_CACHE_PATH>': rlist_cache_path
}
replace_patterns = list(replace_patterns.iteritems())
with open(template_config, 'r') as src_f:
with open(output_config, 'w') as dst_f:
utils.copy_replace(src_f, dst_f, replace_patterns)
# now restore fields added by index_data if required
if restore_fields:
def set_field(field_name, field_value):
cpuvisortls.set_config_field(component_paths['cpuvisor-srv'],
field_name, field_value,
output_config)
set_field('preproc_config.dataset_im_paths', dataset_im_paths)
set_field('preproc_config.dataset_im_base_path',
dataset_im_base_path)
set_field('preproc_config.dataset_feats_file', dataset_feats_file)
def prepare_config():
template_config = os.path.join(templates_dir, 'config.prototxt')
output_config = os.path.join(component_paths['cpuvisor-srv'],
'config.%s.prototxt' % collection['name'])
# if the config file already exists, read in fields added by index_data first
restore_fields = False
if os.path.exists(output_config):
restore_fields = True
def get_field(field_name):
return cpuvisortls.get_config_field(component_paths['cpuvisor-srv'],
field_name,
output_config)
dataset_im_paths = get_field('preproc_config.dataset_im_paths')
dataset_im_base_path = get_field('preproc_config.dataset_im_base_path')
dataset_feats_file = get_field('preproc_config.dataset_feats_file')
# write the new config file
replace_patterns = {
'<MODELS_PATH>': models_path,
'<NEG_IM_PATH>': negimgs_path,
'<NEG_IM_INDEX>': negidx_path,
'<NEG_FEATS_FILE>': negfeats_path,
'<SERVER_ENDPOINT>': server_endpoint,
'<NOTIFY_ENDPOINT>': notify_endpoint,
'<IMAGE_CACHE_PATH>': image_cache_path,
'<RLIST_CACHE_PATH>': rlist_cache_path
}
replace_patterns = list(replace_patterns.iteritems())
with open(template_config, 'r') as src_f:
with open(output_config, 'w') as dst_f:
utils.copy_replace(src_f, dst_f, replace_patterns)
# now restore fields added by index_data if required
if restore_fields:
def set_field(field_name, field_value):
cpuvisortls.set_config_field(component_paths['cpuvisor-srv'],
field_name,
field_value,
output_config)
set_field('preproc_config.dataset_im_paths', dataset_im_paths)
set_field('preproc_config.dataset_im_base_path', dataset_im_base_path)
set_field('preproc_config.dataset_feats_file', dataset_feats_file)
def prepare_limas(base_path, component_cfgs):
component_paths = component_cfgs['components']
links = component_cfgs['links']
collection = component_cfgs['collection']
templates_dir = os.path.join('templates', 'limas')
path = component_paths['limas']
# prepare config
log.info('[limas] Preparing config...')
with utils.change_cwd(component_paths['limas']):
set_env_replace_patterns = [
('<directory to limas>', component_paths['limas']),
('<name of the collection that this instance should host>',
collection['name']),
('<port under which the instance should run>',
str(links['limas']['server_port'])),
('<directory to private data>',
collection['paths']['private_data'])
]
with open('set_env.sh.template', 'r') as src_f:
with open('set_env.sh', 'w') as dst_f:
utils.copy_replace(src_f, dst_f, set_env_replace_patterns)
conf_replace_patterns = {
'<COLLECTION_NAME>':
collection['name'],
'<PATH_TO_WEB_ACCESSIBLE_IMAGES>':
collection['paths']['public_data'],
'<PATH_TO_INDEX_STRUCTURES>':
os.path.join(collection['paths']['index_data'], 'limas'),
'<CPUVISOR_PORT>':
str(links['cpuvisor-srv']['server_port']),
'<CPUVISOR_NOTIFY_PORT>':
str(links['cpuvisor-srv']['notify_port']),
'<URL_TO_COLLECTION_PATH>':
collection['url'],
'<MONGODB_PORT>':
str(links['mongodb']['server_port']),
'<IMSEARCH_PORT>':
str(links['imsearch-tools']['server_port'])
}
conf_replace_patterns = list(conf_replace_patterns.iteritems())
with open(os.path.join('conf', 'conf-template.py'), 'r') as src_f:
with open(os.path.join('conf', collection['name'] + '.py'),
'w') as dst_f:
utils.copy_replace(src_f, dst_f, conf_replace_patterns)
# prepare start script
log.info('[limas] Preparing start script...')
def write_start_script():
outf = os.path.join(path, 'start.sh')
utils.write_template(
templates_dir, 'start.sh', outf, {
'server_port': links['limas']['server_port'],
'base_dir': path,
'name': component_cfgs['collection']['name']
})
os.chmod(outf, 0755)
write_start_script()
