def initialize(self):
		# this only creates the base image; can be updated/cloned later
		base_config_tmpl = os.path.join(BASE_DIR, "tmpl", "docker.tmpl.json")
		base_config = os.path.join(BASE_DIR, "docker.config.json")

		config_keys = [
			DUtilsKeyDefaults['USER_PWD']
		]

		self.config = build_config(config_keys, with_config=base_config_tmpl)
		save_config(self.config, with_config=base_config)

		from dutils.dutils import get_docker_exe, get_docker_ip

		res, self.config = append_to_config({
			'DOCKER_EXE' : get_docker_exe(),
			'DOCKER_IP' : get_docker_ip()
		}, return_config=True, with_config=base_config)

		# build Dockerfile and routine
		dockerfile = os.path.join(BASE_DIR, "tmpl", "Dockerfile.init")
		return build_dockerfile(dockerfile, self.config) and \
			generate_init_routine(self.config, with_config=base_config)
Пример #2
0
    def initialize(self):
        # this only creates the base image; can be updated/cloned later
        base_config_tmpl = os.path.join(BASE_DIR, "tmpl", "docker.tmpl.json")
        base_config = os.path.join(BASE_DIR, "docker.config.json")

        config_keys = [DUtilsKeyDefaults['USER_PWD']]

        self.config = build_config(config_keys, with_config=base_config_tmpl)
        save_config(self.config, with_config=base_config)

        from dutils.dutils import get_docker_exe, get_docker_ip

        res, self.config = append_to_config(
            {
                'DOCKER_EXE': get_docker_exe(),
                'DOCKER_IP': get_docker_ip()
            },
            return_config=True,
            with_config=base_config)

        # build Dockerfile and routine
        dockerfile = os.path.join(BASE_DIR, "tmpl", "Dockerfile.init")
        return build_dockerfile(dockerfile, self.config) and \
         generate_init_routine(self.config, with_config=base_config)
	def build(self):
		def get_task_pool():
			task_pool = []
			task_omits = ["%s.py" % o for o in ["__init__", "evaluate_text", "evaluate_document", "evaluate_file", "pull_from_annex"]]
			annex_tmpl = os.path.join(os.getenv('UNVEILLANCE_BUILD_HOME'), "src", "unveillance", "lib", \
				"Annex", "lib", "Worker", "Tasks")

			for root, _, files in os.walk(annex_tmpl):
				task_pool += ["%(r)s.%(n)s.%(n)s" % { 'r' : root.split("/")[-1], 'n' : f.replace(".py", "") } for f in files if f not in task_omits]

			return task_pool


		config_path = os.path.join(self.config['IMAGE_HOME'], "docker.config.json")
		base_config = load_config(with_config=os.path.join(BASE_DIR, "docker.config.json"))
		
		self.config.update(base_config)
		save_config(self.config, with_config=config_path)

		# allocate published ports
		allocations_manifest = os.path.join(BASE_DIR, "docker.allocations.txt")
		
		last_allocation = FIRST_ALLOCATION - 1

		if os.path.exists(allocations_manifest):
			from fabric.api import settings, local
			with settings(warn_only=True):
				last_allocation = int(local("tail -n 1 %s" % allocations_manifest, capture=True).split(" ")[-1])

		# add frontend configs
		frontend_config = load_config(with_config=os.path.join(BASE_DIR, "tmpl", "frontend.tmpl.json"))
		frontend_config.update({
			'server_host': self.config['DOCKER_IP'],
			'server_port': last_allocation + 1,
			'server_message_port': last_allocation + 2,
			'annex_remote_port' : last_allocation + 3,
			'api.port': last_allocation + 4,	#whatever's available on host
			'annex_local' : os.path.join(self.config['IMAGE_HOME'], "data"),
			'ssh_root' : os.path.join(os.path.expanduser("~"), ".ssh"),
			'task_pool' : get_task_pool()
		})

		# establish PUBLISH_DIRECTIVES for server ports
		publish_directives = {
			"%s" % str(self.config['API_PORT']) : frontend_config['server_port'],
			"%s" % str(self.config['MESSAGE_PORT']) : frontend_config['server_message_port'],
			"22" : frontend_config['annex_remote_port']
		}

		publish_directives_str = ["-p %s:%s" % (str(publish_directives[k]), k) for k in publish_directives.keys()]

		# persist everything
		with open(os.path.join(self.config['IMAGE_HOME'], "unveillance.secrets.json"), 'wb+') as u:
			u.write(json.dumps(frontend_config, indent=4))

		with open(allocations_manifest, 'ab') as u:
			u.write("%s\n" % " ".join([str(frontend_config[c]) for c in \
				['server_port', 'server_message_port', 'annex_remote_port', 'api.port']]))

		res, self.config = append_to_config({
			"DEFAULT_PORTS" : " ".join(publish_directives.keys()),
			"PROJECT_NAME" : os.path.split(self.config['IMAGE_HOME'])[1].replace(" ", "-").lower(),
			"PUBLISH_DIRECTIVES" : " ".join(publish_directives_str)
		}, return_config=True, with_config=config_path)

		dockerfile = os.path.join(BASE_DIR, "tmpl", "Dockerfile.build")
		return build_dockerfile(dockerfile, self.config, dst=self.config['IMAGE_HOME']) and \
			generate_build_routine(self.config, dst=self.config['IMAGE_HOME'])
Пример #4
0
    def build(self):
        def get_task_pool():
            task_pool = []
            task_omits = [
                "%s.py" % o for o in [
                    "__init__", "evaluate_text", "evaluate_document",
                    "evaluate_file", "pull_from_annex"
                ]
            ]
            annex_tmpl = os.path.join(os.getenv('UNVEILLANCE_BUILD_HOME'), "src", "unveillance", "lib", \
             "Annex", "lib", "Worker", "Tasks")

            for root, _, files in os.walk(annex_tmpl):
                task_pool += [
                    "%(r)s.%(n)s.%(n)s" % {
                        'r': root.split("/")[-1],
                        'n': f.replace(".py", "")
                    } for f in files if f not in task_omits
                ]

            return task_pool

        config_path = os.path.join(self.config['IMAGE_HOME'],
                                   "docker.config.json")
        base_config = load_config(
            with_config=os.path.join(BASE_DIR, "docker.config.json"))

        self.config.update(base_config)
        save_config(self.config, with_config=config_path)

        # allocate published ports
        allocations_manifest = os.path.join(BASE_DIR, "docker.allocations.txt")

        last_allocation = FIRST_ALLOCATION - 1

        if os.path.exists(allocations_manifest):
            from fabric.api import settings, local
            with settings(warn_only=True):
                last_allocation = int(
                    local("tail -n 1 %s" % allocations_manifest,
                          capture=True).split(" ")[-1])

        # add frontend configs
        frontend_config = load_config(
            with_config=os.path.join(BASE_DIR, "tmpl", "frontend.tmpl.json"))
        frontend_config.update({
            'server_host':
            self.config['DOCKER_IP'],
            'server_port':
            last_allocation + 1,
            'server_message_port':
            last_allocation + 2,
            'annex_remote_port':
            last_allocation + 3,
            'api.port':
            last_allocation + 4,  #whatever's available on host
            'annex_local':
            os.path.join(self.config['IMAGE_HOME'], "data"),
            'ssh_root':
            os.path.join(os.path.expanduser("~"), ".ssh"),
            'task_pool':
            get_task_pool()
        })

        # establish PUBLISH_DIRECTIVES for server ports
        publish_directives = {
            "%s" % str(self.config['API_PORT']):
            frontend_config['server_port'],
            "%s" % str(self.config['MESSAGE_PORT']):
            frontend_config['server_message_port'],
            "22":
            frontend_config['annex_remote_port']
        }

        publish_directives_str = [
            "-p %s:%s" % (str(publish_directives[k]), k)
            for k in publish_directives.keys()
        ]

        # persist everything
        with open(
                os.path.join(self.config['IMAGE_HOME'],
                             "unveillance.secrets.json"), 'wb+') as u:
            u.write(json.dumps(frontend_config, indent=4))

        with open(allocations_manifest, 'ab') as u:
            u.write("%s\n" % " ".join([str(frontend_config[c]) for c in \
             ['server_port', 'server_message_port', 'annex_remote_port', 'api.port']]))

        res, self.config = append_to_config(
            {
                "DEFAULT_PORTS":
                " ".join(publish_directives.keys()),
                "PROJECT_NAME":
                os.path.split(self.config['IMAGE_HOME'])[1].replace(
                    " ", "-").lower(),
                "PUBLISH_DIRECTIVES":
                " ".join(publish_directives_str)
            },
            return_config=True,
            with_config=config_path)

        dockerfile = os.path.join(BASE_DIR, "tmpl", "Dockerfile.build")
        return build_dockerfile(dockerfile, self.config, dst=self.config['IMAGE_HOME']) and \
         generate_build_routine(self.config, dst=self.config['IMAGE_HOME'])