def initialize(self):
# this only creates the base image; can be updated/cloned later
base_config_tmpl = os.path.join(BASE_DIR, "tmpl", "docker.tmpl.json")
base_config = os.path.join(BASE_DIR, "docker.config.json")
config_keys = [
DUtilsKeyDefaults['USER_PWD']
]
self.config = build_config(config_keys, with_config=base_config_tmpl)
save_config(self.config, with_config=base_config)
from dutils.dutils import get_docker_exe, get_docker_ip
res, self.config = append_to_config({
'DOCKER_EXE' : get_docker_exe(),
'DOCKER_IP' : get_docker_ip()
}, return_config=True, with_config=base_config)
# build Dockerfile and routine
dockerfile = os.path.join(BASE_DIR, "tmpl", "Dockerfile.init")
return build_dockerfile(dockerfile, self.config) and \
generate_init_routine(self.config, with_config=base_config)
def initialize(self):
# this only creates the base image; can be updated/cloned later
base_config_tmpl = os.path.join(BASE_DIR, "tmpl", "docker.tmpl.json")
base_config = os.path.join(BASE_DIR, "docker.config.json")
config_keys = [DUtilsKeyDefaults['USER_PWD']]
self.config = build_config(config_keys, with_config=base_config_tmpl)
save_config(self.config, with_config=base_config)
from dutils.dutils import get_docker_exe, get_docker_ip
res, self.config = append_to_config(
{
'DOCKER_EXE': get_docker_exe(),
'DOCKER_IP': get_docker_ip()
},
return_config=True,
with_config=base_config)
# build Dockerfile and routine
dockerfile = os.path.join(BASE_DIR, "tmpl", "Dockerfile.init")
return build_dockerfile(dockerfile, self.config) and \
generate_init_routine(self.config, with_config=base_config)
def build(self):
def get_task_pool():
task_pool = []
task_omits = ["%s.py" % o for o in ["__init__", "evaluate_text", "evaluate_document", "evaluate_file", "pull_from_annex"]]
annex_tmpl = os.path.join(os.getenv('UNVEILLANCE_BUILD_HOME'), "src", "unveillance", "lib", \
"Annex", "lib", "Worker", "Tasks")
for root, _, files in os.walk(annex_tmpl):
task_pool += ["%(r)s.%(n)s.%(n)s" % { 'r' : root.split("/")[-1], 'n' : f.replace(".py", "") } for f in files if f not in task_omits]
return task_pool
config_path = os.path.join(self.config['IMAGE_HOME'], "docker.config.json")
base_config = load_config(with_config=os.path.join(BASE_DIR, "docker.config.json"))
self.config.update(base_config)
save_config(self.config, with_config=config_path)
# allocate published ports
allocations_manifest = os.path.join(BASE_DIR, "docker.allocations.txt")
last_allocation = FIRST_ALLOCATION - 1
if os.path.exists(allocations_manifest):
from fabric.api import settings, local
with settings(warn_only=True):
last_allocation = int(local("tail -n 1 %s" % allocations_manifest, capture=True).split(" ")[-1])
# add frontend configs
frontend_config = load_config(with_config=os.path.join(BASE_DIR, "tmpl", "frontend.tmpl.json"))
frontend_config.update({
'server_host': self.config['DOCKER_IP'],
'server_port': last_allocation + 1,
'server_message_port': last_allocation + 2,
'annex_remote_port' : last_allocation + 3,
'api.port': last_allocation + 4, #whatever's available on host
'annex_local' : os.path.join(self.config['IMAGE_HOME'], "data"),
'ssh_root' : os.path.join(os.path.expanduser("~"), ".ssh"),
'task_pool' : get_task_pool()
})
# establish PUBLISH_DIRECTIVES for server ports
publish_directives = {
"%s" % str(self.config['API_PORT']) : frontend_config['server_port'],
"%s" % str(self.config['MESSAGE_PORT']) : frontend_config['server_message_port'],
"22" : frontend_config['annex_remote_port']
}
publish_directives_str = ["-p %s:%s" % (str(publish_directives[k]), k) for k in publish_directives.keys()]
# persist everything
with open(os.path.join(self.config['IMAGE_HOME'], "unveillance.secrets.json"), 'wb+') as u:
u.write(json.dumps(frontend_config, indent=4))
with open(allocations_manifest, 'ab') as u:
u.write("%s\n" % " ".join([str(frontend_config[c]) for c in \
['server_port', 'server_message_port', 'annex_remote_port', 'api.port']]))
res, self.config = append_to_config({
"DEFAULT_PORTS" : " ".join(publish_directives.keys()),
"PROJECT_NAME" : os.path.split(self.config['IMAGE_HOME'])[1].replace(" ", "-").lower(),
"PUBLISH_DIRECTIVES" : " ".join(publish_directives_str)
}, return_config=True, with_config=config_path)
dockerfile = os.path.join(BASE_DIR, "tmpl", "Dockerfile.build")
return build_dockerfile(dockerfile, self.config, dst=self.config['IMAGE_HOME']) and \
generate_build_routine(self.config, dst=self.config['IMAGE_HOME'])
def build(self):
def get_task_pool():
task_pool = []
task_omits = [
"%s.py" % o for o in [
"__init__", "evaluate_text", "evaluate_document",
"evaluate_file", "pull_from_annex"
]
]
annex_tmpl = os.path.join(os.getenv('UNVEILLANCE_BUILD_HOME'), "src", "unveillance", "lib", \
"Annex", "lib", "Worker", "Tasks")
for root, _, files in os.walk(annex_tmpl):
task_pool += [
"%(r)s.%(n)s.%(n)s" % {
'r': root.split("/")[-1],
'n': f.replace(".py", "")
} for f in files if f not in task_omits
]
return task_pool
config_path = os.path.join(self.config['IMAGE_HOME'],
"docker.config.json")
base_config = load_config(
with_config=os.path.join(BASE_DIR, "docker.config.json"))
self.config.update(base_config)
save_config(self.config, with_config=config_path)
# allocate published ports
allocations_manifest = os.path.join(BASE_DIR, "docker.allocations.txt")
last_allocation = FIRST_ALLOCATION - 1
if os.path.exists(allocations_manifest):
from fabric.api import settings, local
with settings(warn_only=True):
last_allocation = int(
local("tail -n 1 %s" % allocations_manifest,
capture=True).split(" ")[-1])
# add frontend configs
frontend_config = load_config(
with_config=os.path.join(BASE_DIR, "tmpl", "frontend.tmpl.json"))
frontend_config.update({
'server_host':
self.config['DOCKER_IP'],
'server_port':
last_allocation + 1,
'server_message_port':
last_allocation + 2,
'annex_remote_port':
last_allocation + 3,
'api.port':
last_allocation + 4, #whatever's available on host
'annex_local':
os.path.join(self.config['IMAGE_HOME'], "data"),
'ssh_root':
os.path.join(os.path.expanduser("~"), ".ssh"),
'task_pool':
get_task_pool()
})
# establish PUBLISH_DIRECTIVES for server ports
publish_directives = {
"%s" % str(self.config['API_PORT']):
frontend_config['server_port'],
"%s" % str(self.config['MESSAGE_PORT']):
frontend_config['server_message_port'],
"22":
frontend_config['annex_remote_port']
}
publish_directives_str = [
"-p %s:%s" % (str(publish_directives[k]), k)
for k in publish_directives.keys()
]
# persist everything
with open(
os.path.join(self.config['IMAGE_HOME'],
"unveillance.secrets.json"), 'wb+') as u:
u.write(json.dumps(frontend_config, indent=4))
with open(allocations_manifest, 'ab') as u:
u.write("%s\n" % " ".join([str(frontend_config[c]) for c in \
['server_port', 'server_message_port', 'annex_remote_port', 'api.port']]))
res, self.config = append_to_config(
{
"DEFAULT_PORTS":
" ".join(publish_directives.keys()),
"PROJECT_NAME":
os.path.split(self.config['IMAGE_HOME'])[1].replace(
" ", "-").lower(),
"PUBLISH_DIRECTIVES":
" ".join(publish_directives_str)
},
return_config=True,
with_config=config_path)
dockerfile = os.path.join(BASE_DIR, "tmpl", "Dockerfile.build")
return build_dockerfile(dockerfile, self.config, dst=self.config['IMAGE_HOME']) and \
generate_build_routine(self.config, dst=self.config['IMAGE_HOME'])
