Esempio n. 1
0
class Two1ComposerContainers(Two1Composer):
    """ Manage machine-payable microservices in containers.
    """

    def __init__(self):
        self._connected = ComposerState.DISCONNECTED
        self.provider = TwentyOneProvider()
        self.default_wallet = Two1Wallet(self.wallet_file, self.provider)

    class ServiceManager:
        """ Query and modify user services persisting at cls.USER_SERVICES_FILE.
        """

        USER_SERVICES_FILE = os.path.join(Two1Composer.BASE_DIR, "user-services.json")

        class Image(namedtuple('Image', 'docker_hub_account repository tag')):

            def _asdict(self):
                # Fixes a bug for Python 3.4 users
                # https://bugs.python.org/issue24931
                'A new OrderedDict mapping field names to their values'
                return OrderedDict(zip(self._fields, self))

            @property
            def is_dockerhub_image(self):
                """ Returns: True iff Image instance has all fields.
                """
                return self.docker_hub_account and self.repository and self.tag

            @property
            def is_local_image(self):
                """ Returns: True iff Image instance doesn't have docker_hub_account but has all other fields.
                """
                return not self.docker_hub_account and self.repository and self.tag

            def __str__(self):
                """ Returns: Docker image name constructed from Image instance fields.
                """
                if self.is_dockerhub_image:
                    return '%s/%s:%s' % (self.docker_hub_account, self.repository, self.tag)
                elif self.is_local_image:
                    return '%s:%s' % (self.repository, self.tag)
                else:
                    raise ValueError()

            @classmethod
            def from_string(cls, image_name):
                """ Constructs an Image instance from a docker image name.

                Args:
                    image_name (str): A docker image name.

                Returns:
                    Image: An Image instance.
                """
                slashes = re.findall('/', image_name)
                colons = re.findall(':', image_name)

                if len(slashes) == 1:
                    if len(colons) == 1 and image_name.find('/') < image_name.find(':'):
                        docker_hub_account, rest = image_name.split('/')
                        repository, tag = rest.split(':')
                        return cls(docker_hub_account=docker_hub_account, repository=repository, tag=tag)
                    elif len(colons) == 0:
                        docker_hub_account, repository = image_name.split('/')
                        return cls(docker_hub_account=docker_hub_account, repository=repository, tag='latest')
                elif len(slashes) == 0:
                    if len(colons) == 1:
                        repository, tag = image_name.split(':')
                        return cls(docker_hub_account=None, repository=repository, tag=tag)
                    elif len(colons) == 0:
                        return cls(docker_hub_account=None, repository=image_name, tag='latest')
                raise ValueError()

        @classmethod
        def get_image(cls, service_name):
            """ Constructs an Image instance for a service.

            Args:
                service_name (str): The name of either a 21 service in the 21dotco/two1 repository or a user service
                                    added to ServiceManager.USER_SERVICES_FILE by ServiceManager.add_service.

            Returns:
                Image: An Image instance corresponding to the given service.
            """
            if service_name in cls.available_21_services():
                return cls.Image(
                    docker_hub_account='21dotco',
                    repository='two1',
                    tag=service_name if service_name in Two1Composer.BASE_SERVICES else 'service-%s' % service_name
                )
            elif service_name in cls.available_user_services():
                return cls.Image(**cls._get_user_service_dict()[service_name])
            else:
                raise ValueError()

        @classmethod
        def available_services(cls):
            """ Returns: All available service names.
            """
            return cls.available_21_services() | cls.available_user_services()

        @classmethod
        def available_21_services(cls):
            """ Returns: All available 21 services by querying Docker Hub.
            """
            service_image_data = requests.get(os.path.join(
                Two1Composer.DOCKERHUB_API_URL, Two1Composer.DOCKERHUB_REPO, 'tags')).json().get('results')
            return set([image_data['name'].split('service-')[1] for image_data in
                        service_image_data if re.match(r'^service-', image_data['name'])])

        @classmethod
        def available_user_services(cls):
            """ Returns: All available user services.
            """
            return set(cls._get_user_service_dict().keys())

        @classmethod
        def add_service(cls, service_name, image_name_string,
                        service_successfully_added_hook, service_already_exists_hook,
                        service_failed_to_add_hook):
            """ Adds a new service definition to ServiceManager.USER_SERVICES_FILE.

            Args:
                service_name (str): Name of the service definition to add.
                image_name_string (str): Docker image name for the service definition.
            """
            service_dict = cls._get_user_service_dict()
            if service_name in service_dict:
                service_already_exists_hook(service_name)
            else:
                service_dict[service_name] = cls.Image.from_string(image_name_string)._asdict()
                if cls._commit_user_service_dict(service_dict):
                    service_successfully_added_hook(service_name)
                else:
                    service_failed_to_add_hook(service_name)

        @classmethod
        def remove_service(cls, service_name,
                           service_successfully_removed_hook,
                           service_does_not_exists_hook,
                           service_failed_to_remove_hook):
            """ Removes a service definition from ServiceManager.USER_SERVICES_FILE.

            Args:
                service_name (str): Name of the service definition to remove.
            """
            service_dict = cls._get_user_service_dict()
            if service_name in service_dict:
                del service_dict[service_name]
                if cls._commit_user_service_dict(service_dict):
                    service_successfully_removed_hook(service_name)
                else:
                    service_failed_to_remove_hook(service_name)
            else:
                service_does_not_exists_hook(service_name)

        @classmethod
        def _get_user_service_dict(cls):
            """ Returns: ServiceManager.USER_SERVICES_FILE as a dict.
            """
            try:
                with open(cls.USER_SERVICES_FILE, 'r') as data_file:
                    service_dict = json.load(data_file)
            except:
                return {}
            else:
                return service_dict

        @classmethod
        def _commit_user_service_dict(cls, service_dict):
            """ Writes a dict of user services to ServiceManager.USER_SERVICES_FILE in json format.

            Args:
                service_dict (dict): A dictionary of user services of the form
                                     {service_name : _as_dict representation of corresponding Image instance..}.

            Returns:
                bool: True iff no exceptions were raised when writing service_dict to ServiceManager.USER_SERVICES_FILE
                      as json.
            """
            try:
                with open(cls.USER_SERVICES_FILE, 'w') as outfile:
                    json.dump(service_dict, outfile)
            except:
                return False
            else:
                return True

    class ComposerYAMLContext(YamlDataContext):
        """ Context manager for composer YAML service file.
        """

        def __init__(self, username=None, password=None, server_port=None, mnemonic=None):
            self.username = username
            self.password = password
            self.server_port = server_port
            self.mnemonic = mnemonic
            super().__init__(Two1Composer.COMPOSE_FILE)

        def __enter__(self):
            sup = super().__enter__()
            for service in self.data['services']:
                service_definition = self.data['services'][service]
                if 'environment' in service_definition:

                    if 'TWO1_USERNAME' in service_definition['environment'] and self.username is not None:
                        service_definition['environment']['TWO1_USERNAME'] = self.username

                    if 'TWO1_PASSWORD' in service_definition['environment'] and self.password is not None:
                        service_definition['environment']['TWO1_PASSWORD'] = self.password

                    if 'TWO1_WALLET_MNEMONIC' in service_definition['environment'] and self.mnemonic is not None:
                        service_definition['environment']['TWO1_WALLET_MNEMONIC'] = self.mnemonic

                    if 'PAYMENT_SERVER_IP' in service_definition['environment'] and self.server_port is not None:
                        rest = service_definition['environment']['PAYMENT_SERVER_IP'].rsplit(':', maxsplit=1)[-1]
                        service_definition['environment']['PAYMENT_SERVER_IP'] = '%s:%s' % (rest, self.server_port)
            return sup

        def _filler(self):
            """ Create the base service description file.
            """
            return {
                'version': '2',
                'services': {
                    'base': {
                        'image': '%s:base' % Two1Composer.DOCKERHUB_REPO,
                    },
                    'router': {
                        'image': '%s:router' % Two1Composer.DOCKERHUB_REPO,
                        'container_name': 'sell_router',
                        'restart': 'always',
                        'volumes': [
                            Two1Composer.SITES_ENABLED_PATH + ":/etc/nginx/sites-enabled",
                            Two1Composer.SITES_AVAILABLE_PATH + ":/etc/nginx/sites-available",
                        ],
                        'ports': ['%s:%s' % (self.server_port, self.server_port)],
                        'links': [
                            'payments:payments',
                        ],
                    },
                    'payments': {
                        'image': '%s:payments' % Two1Composer.DOCKERHUB_REPO,
                        'depends_on': ['base'],
                        'container_name': 'sell_payments',
                        'restart': 'always',
                        'environment': {
                            "TWO1_USERNAME": str(self.username),
                            "TWO1_PASSWORD": str(self.password),
                            "TWO1_WALLET_MNEMONIC": str(self.mnemonic)
                        },
                        'volumes': [
                            Two1Composer.DB_DIR + ":/usr/src/db/"
                        ],
                        'logging': {
                            'driver': 'json-file'
                        },
                        'cap_drop': [
                            'ALL'
                        ],
                        'cap_add': [
                            'DAC_OVERRIDE',
                            'NET_RAW',
                        ],
                    }
                }
            }

    # public api
    def connect(self, machine_env, host, machine_config_file):
        """ Connect service composer to machine layer.

        Args:
            machine_env (dict): Environment dictionary for the docker client of the machine layer.
            host (str): Hostname of the machine layer docker daemon.
            machine_config_file (str): Path to the config file for the machine layer.
        """
        self.machine_env = machine_env
        self.machine_host = host
        with open(machine_config_file, 'r') as f:
            self.machine_config = json.load(f)
        self.docker_client = Client(**docker_env(assert_hostname=False,
                                                 environment=self.machine_env))
        self._connected = ComposerState.CONNECTED

    def initialize_server(self, username, password, server_port, wallet=None):
        """ Initialize micropayments server.

        Define boilerplate services, networks, and volumes composer file
        and nginx server config.

        Generates a wallet mnemonic if non-existent.

        Args:
            username (str): Username to log in with.
            password (str): Password to log in with.
            server_port (int): The server port that the router is running on.
            wallet: The wallet to use for the payments server and subsequent services.
        """
        self._create_base_server(server_port)  # create base router server config
        self._create_payments_route()  # create route to payments server

        new_wallet = None  # rv[1], not None if mnemonic is replaced in this function

        # generate service description (yaml)
        with self.ComposerYAMLContext(username, password, server_port) as composer_yaml:
            try:
                mnemonic = composer_yaml['services']['payments']['environment']['TWO1_WALLET_MNEMONIC']
                if not mnemonic or mnemonic == str(None):  # if mnemonic is Falsy or uninitialized
                    raise ValueError()
            except (KeyError, ValueError):  # catches if mnemonic is Falsy or doesn't exist in dict tree
                new_machine_wallet = self.default_wallet.create(self.provider)[1]
                composer_yaml['services']['payments']['environment']['TWO1_WALLET_MNEMONIC'] = new_machine_wallet
                new_wallet = new_machine_wallet

        return 0, new_wallet

    def pull_image(self, image,
                   image_sucessfully_pulled_hook, image_failed_to_pull_hook, image_is_local_hook,
                   image_is_malformed_hook):
        """ Pulls an Image instance iff it is a Docker Hub image.

        Args:
            image (Image): An Image instance.
        """
        if image.is_dockerhub_image:
            try:
                self.docker_client.pull('%s/%s' % (image.docker_hub_account, image.repository),
                                        tag=image.tag, stream=False)
            except:
                image_failed_to_pull_hook(image)
            else:
                image_sucessfully_pulled_hook(image)
        elif image.is_local_image:
            image_is_local_hook(image)
        else:
            image_is_malformed_hook(image)

    def start_services(self, service_names,
                       failed_to_start_hook, started_hook, failed_to_restart_hook, restarted_hook, failed_to_up_hook,
                       up_hook):
        """ Start selected services.

        Args:
            service_names (list): List of service names to start.
            failed_to_start_hook (Callable): A callable hook that takes in a service name and is run when said service
                                             fails to start.
            started_hook (Callable): A callable hook that takes in a service name and is run when said service starts.
            failed_to_restart_hook (Callable): A callable hook that takes in a service name and is run when said service
                                               fails to restart.
            restarted_hook (Callable): A callable hook that takes in a service name and is run when said service
                                       restarts.
            failed_to_up_hook (Callable): A callable hook that takes in a service name and is run when said service
                                          fails to go up.
            up_hook (Callable): A callable hook that takes in a service name and is run when said service goes up.
        """
        self._start_sell_service('base', failed_to_start_hook, started_hook, failed_to_up_hook, up_hook)
        self._start_sell_service('router', failed_to_start_hook, started_hook, failed_to_up_hook, up_hook)
        self._start_sell_service('payments', failed_to_start_hook, started_hook, failed_to_up_hook, up_hook)

        self._restart_sell_service('router', failed_to_start_hook, started_hook, failed_to_restart_hook, restarted_hook,
                                   failed_to_up_hook, up_hook)

        # Attempt to start all market services
        for service_name in service_names:
            try:
                image = self.ServiceManager.get_image(service_name)
                container_name = self.service_name_2_container_name(service_name)

                # create nginx routes for service_name
                self._create_service_route(service_name)
                # add service_name to docker compose file
                with self.ComposerYAMLContext() as docker_compose_yaml:
                    username = docker_compose_yaml['services']['payments']['environment']['TWO1_USERNAME']
                    password = docker_compose_yaml['services']['payments']['environment']['TWO1_PASSWORD']
                    mnemonic = docker_compose_yaml['services']['payments']['environment']['TWO1_WALLET_MNEMONIC']
                    docker_compose_yaml['services'][service_name] = {
                        'image': str(image),
                        'container_name': container_name,
                        'depends_on': ['base'],
                        'restart': 'always',
                        'environment': {
                            "TWO1_USERNAME": str(username),
                            "TWO1_PASSWORD": str(password),
                            "TWO1_WALLET_MNEMONIC": str(mnemonic),
                            "SERVICE": str(service_name),
                            "PAYMENT_SERVER_IP":
                                "http://%s:%s" % (self.machine_host, self.machine_config["server_port"])
                        },
                        'volumes': [
                            Two1Composer.DB_DIR + ":/usr/src/db/"
                        ],
                        'logging': {
                            'driver': 'json-file'
                        },
                        'cap_drop': [
                            'ALL'
                        ],
                        'cap_add': [
                            'DAC_OVERRIDE',
                            'NET_RAW',
                        ],
                    }
                    link_str = '%s:%s' % (service_name, service_name)
                    if link_str not in docker_compose_yaml['services']['router']['links']:
                        docker_compose_yaml['services']['router']['links'].append(link_str)
            except:
                # something went wrong while configuring service_name
                failed_to_start_hook(service_name)
            else:
                # attempt to build service_name
                self._start_sell_service(service_name, failed_to_start_hook, started_hook, failed_to_up_hook, up_hook)

        self._restart_sell_service('router', failed_to_start_hook, started_hook, failed_to_restart_hook, restarted_hook,
                                   failed_to_up_hook, up_hook)

    def _start_sell_service(self, service_name, failed_to_start_hook, started_hook, failed_to_up_hook, up_hook,
                            timeout=Two1Composer.SERVICE_START_TIMEOUT):
        try:
            subprocess.check_output(["docker-compose", "-f", Two1Composer.COMPOSE_FILE, "up", "-d", service_name],
                                    stderr=subprocess.DEVNULL, env=self.machine_env)
        except subprocess.CalledProcessError:
            failed_to_start_hook(service_name)
        else:
            started_hook(service_name)
            if service_name == 'router':
                time.sleep(5)
            elif service_name != 'router' and service_name != 'base':
                start = time.clock()

                exec_id = self.docker_client.exec_create('sell_router', "curl %s:5000" % service_name)['Id']
                self.docker_client.exec_start(exec_id)
                running = True

                while time.clock() - start < timeout and running is True:
                    running = self.docker_client.exec_inspect(exec_id)['Running']

                if running is True:
                    failed_to_up_hook(service_name)
                else:
                    up_hook(service_name)

    def _restart_sell_service(self, service_name, failed_to_start_hook, started_hook, failed_to_restart_hook,
                              restarted_hook, failed_to_up_hook, up_hook):
        try:
            self.docker_client.stop("sell_%s" % service_name)
        except:
            is_restart = False
        else:
            is_restart = True

        self._start_sell_service(service_name, failed_to_restart_hook if is_restart else failed_to_start_hook,
                                 restarted_hook if is_restart else started_hook, failed_to_up_hook, up_hook)

    def stop_services(self, service_names,
                      service_found_stopped_and_removed_hook,
                      service_failed_to_stop_hook,
                      service_failed_to_be_removed_hook,
                      service_not_found_hook):
        """ Stop selected services and remove containers.

        Args:
            service_names (set): Set of services to stop.
            service_found_stopped_and_removed_hook (Callable): A callable hook that takes in a service name and is run
                                                               when said service is found, stopped, and removed.
            service_failed_to_stop_hook (Callable): A callable hook that takes in a service name and is run when said
                                                    service fails to be stopped.
            service_failed_to_be_removed_hook (Callable): A callable hook that takes in a service name and is run when
                                                          said service fails to be removed.
            service_not_found_hook (Callable): A callable hook that takes in a service name and is run when said service
                                               isn't found.

        """
        for service_name in service_names:
            if service_name in self.get_running_services():
                container_name = self.service_name_2_container_name(service_name)
                try:
                    self.docker_client.stop(container_name)
                except:
                    service_failed_to_stop_hook(service_name)
                else:
                    try:
                        self.docker_client.remove_container(container_name)
                    except:
                        service_failed_to_be_removed_hook(service_name)
                    else:
                        service_found_stopped_and_removed_hook(service_name)
            else:
                service_not_found_hook(service_name)

    def silently_force_stop_all_services(self):
        running_container_names = self.docker_client.containers(filters={"status": "running"})
        for container_name in running_container_names:
            self.docker_client.remove_container(container_name, force=True)

    @staticmethod
    def container_names_2_service_names(container_definitions):
        """ Return service names from container definitions.

        See service_name_2_container_name for the inverse operation but on one service name.

        Args:
            container_definitions (list): List of container descriptions as returned by self.docker_client.containers.

        Returns:
            set: Set of service names generated by removing the 'sell_' prefix from the containers' names.
        """
        return set([container_definition['Names'][0][6:] for container_definition in container_definitions])

    @staticmethod
    def service_name_2_container_name(service_name):
        """ Generates a container name from a service name by prepending 'sell_'
        """
        return 'sell_%s' % service_name

    def status_services(self, services):
        """ Gets running status of specified services.

        Args:
            services (list): List of services to get status for.
        """

        existent_services = self.get_services(all=True)
        running_services = self.get_services(filters={"status": "running"})
        exited_services = self.get_services(filters={"status": "exited"})

        return {
            "running": running_services & services,
            "exited": exited_services & services,
            "nonexistent": services - existent_services
        }

    def get_services(self, *args, **kwargs):
        """ Call docker_client.containers | convert resulting container names to service names | remove base services
        """
        return self.container_names_2_service_names(
            self.docker_client.containers(*args, **kwargs)
        ) - Two1Composer.BASE_SERVICES

    def get_running_services(self):
        """ Get list of currently running services that aren't 21 base services.

        Returns:
            set: Set of currently running services.
        """
        return self.get_services(filters={"status": "running"})

    def status_router(self, service_running_hook, service_unknown_state_hook):
        """ Get status of Nginx router container.

        Args:
            service_running_hook (Callable): A callable hook that takes in a service name and is run when said service
                                             is running.
            service_unknown_state_hook (Callable): A callable hook that takes in a service name and is run when said
                                                   service is in an unknown state.
        """
        if len(self.docker_client.containers(all=True, filters={"name": "sell_router", "status": "running"})) == 1:
            service_running_hook("router")
        else:
            service_unknown_state_hook("router")

    def status_payments_server(self, service_running_hook, service_unknown_state_hook):
        """ Get status of payment channels server.

        Args:
            service_running_hook (Callable): A callable hook that takes in a service name and is run when said service
                                             is running.
            service_unknown_state_hook (Callable): A callable hook that takes in a service name and is run when said
                                                   service is in an unknown state.
        """
        if len(self.docker_client.containers(all=True, filters={"name": "sell_payments", "status": "running"})) == 1:
            service_running_hook("payments")
        else:
            service_unknown_state_hook("payments")

    @staticmethod
    def _create_base_server(server_port):
        """ Create nginx base server config.

        Args:
            server_port (int): port for 21 sell server.
        """
        try:
            # create nginx router dirs
            shutil.rmtree(Two1Composer.SITES_ENABLED_PATH, ignore_errors=True)
            shutil.rmtree(Two1Composer.SITES_AVAILABLE_PATH, ignore_errors=True)
            os.makedirs(Two1Composer.SITES_ENABLED_PATH, exist_ok=True)
            os.makedirs(Two1Composer.SITES_AVAILABLE_PATH, exist_ok=True)

            # create base nginx server
            with open(os.path.join(Two1Composer.SITES_ENABLED_PATH,
                                   "two1baseserver"), 'w') as f:
                f.write("server {\n"
                        "    listen " + str(server_port) + ";\n"
                        "    include /etc/nginx/sites-available/*;\n"
                        "}\n"
                        )
        except Exception:
            raise exceptions.Two1ComposerServiceDefinitionException()

    @staticmethod
    def _create_service_route(service):
        """ Create route for container service.
        """
        os.makedirs(Two1Composer.SITES_AVAILABLE_PATH, exist_ok=True)
        try:
            with open(os.path.join(Two1Composer.SITES_AVAILABLE_PATH, service), 'w') as f:
                f.write("location /" + service + " {\n"
                        "    rewrite ^/" + service + "/?(.*) /$1 break;\n"
                        "    proxy_pass http://" + service + ":" + str(5000) + ";\n"
                        "    proxy_set_header Host $host;\n"
                        "    proxy_set_header X-Real-IP $remote_addr;\n"
                        "    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"
                        "}\n")
        except Exception:
            raise exceptions.Two1ComposerRouteException()

    @staticmethod
    def _create_payments_route():
        """ Add route to payments server.
        """
        os.makedirs(Two1Composer.SITES_AVAILABLE_PATH, exist_ok=True)
        try:
            # write nginx route for payments server
            with open(os.path.join(Two1Composer.SITES_AVAILABLE_PATH, "payments"), 'w') as f:
                f.write("location /payment {\n"
                        "    proxy_pass http://payments:" + str(5000) + ";\n"
                        "    proxy_set_header Host $host;\n"
                        "    proxy_set_header X-Real-IP $remote_addr;\n"
                        "    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"
                        "}\n")
        except Exception:
            raise exceptions.Two1ComposerRouteException()

    def publish_service(self, service_name, host_override, rest_client, published_hook,
                        already_published_hook, failed_to_publish_hook,
                        unknown_publish_error_hook):
        strm, stat = self.docker_client.get_archive('sell_%s' % service_name,
                                                    '/usr/src/app/manifest.yaml')

        with tarfile.open(fileobj=BytesIO(strm.read()), mode='r') as tf:
            manifest = yaml.load(tf.extractfile(stat[u'name']).read().decode())
        manifest['host'] = host_override

        try:
            resp = rest_client.publish({"manifest": manifest,
                                        "marketplace": "21mkt"})
        except ServerRequestError as e:
            if e.status_code == 403 and e.data.get("error") == "TO600":
                already_published_hook(service_name)
            else:
                failed_to_publish_hook(service_name)
        except:
            unknown_publish_error_hook(service_name)
        else:
            if resp.status_code == 201:
                published_hook(service_name)
            else:
                failed_to_publish_hook(service_name)

    def read_server_config(self):
        try:
            with open(Two1Composer.COMPOSE_FILE) as f:
                return yaml.load(f)

        except FileNotFoundError:
            return {}

    def get_services_mnemonic(self):
        if os.path.isfile(Two1Composer.COMPOSE_FILE):
            with self.ComposerYAMLContext() as composer_yaml:
                try:
                    maybe_mnemonic = composer_yaml['services']['payments']['environment']['TWO1_WALLET_MNEMONIC']
                except KeyError:
                    rv = None
                else:
                    rv = maybe_mnemonic
        else:
            rv = None
        return rv
Esempio n. 2
0
    key_container = client.create_container(
        "johncclayton/syncthing-cli",
        "/generate-identities.sh",
        environment={"KEY_HOSTS": " ".join(HOSTS.keys())})
    print("Generating keys...")
    key_response = client.start(container=key_container.get('Id'))
    client.wait(container=key_container.get('Id'))

    # now copy the files from within the container to the local disk
    if os.path.exists(key_pathname):
        shutil.rmtree(key_pathname)

    os.mkdir(key_pathname)

    print("Copying key data from container...")
    strm, stat = client.get_archive(key_container, "/keys")
    data = io.BytesIO(strm.read())
    tarobj = tarfile.open(fileobj=data)
    tarobj.extractall(path=os.getcwd())

# find the syncthing config.xml template and populate it with all the known device Ids.  This
# file is then used within all the syncthing images to ensure that all hosts know about each other.
env = Environment(
    loader=FileSystemLoader(os.path.join(os.getcwd(), "templates")))
config_template = env.get_template('config_xml.tpl')

devices = []

print("Creating config.xml for each device...")

# create a single config.xml file that has the ID values of every known key
Esempio n. 3
0
class Two1ComposerContainers(Two1Composer):
    """ Manage machine-payable microservices in containers.
    """

    def __init__(self):
        self._connected = ComposerState.DISCONNECTED
        self.provider = TwentyOneProvider()
        self.default_wallet = Two1Wallet(self.wallet_file, self.provider)

    # public api
    def connect(self, machine_env, host, machine_config_file):
        """ Connect service composer to machine layer.

        Args:
            machine_env (dict): Environment dictionary for the docker client of the machine layer
            host: Hostname of the machine layer docker daemon
            machine_config_file (str): Path to the config file for the machine layer
        """
        self.machine_env = machine_env
        self.machine_host = host
        with open(machine_config_file, 'r') as f:
            self.machine_config = json.load(f)
        self.docker_client = Client(**docker_env(assert_hostname=False,
                                                 environment=self.machine_env))
        self._connected = ComposerState.CONNECTED

    def initialize_server(self, username, password, server_port, wallet=None):
        """ Initialize micropayments server.

        Define boilerplate services, networks, and volumes composer file
        and nginx server config.

        Generates a wallet mnemonic if non-existent.

        Args:
            username (str): Username to log in with
            password (str): Password to log in with
            server_port (int): The server port that the router is running on
            wallet: The wallet to use for the payments server and subsequent services
        """
        self._create_base_server(server_port)  # create base router server config
        self._create_payments_route()  # create route to payments server

        new_wallet = None  # rv[1], not None if mnemonic is replaced in this function

        # generate service description (yaml)
        with self.ComposerYAMLContext(username, password, server_port) as composer_yaml:
            try:
                mnemonic = composer_yaml['services']['payments']['environment']['TWO1_WALLET_MNEMONIC']
                if not mnemonic or mnemonic == str(None):  # if mnemonic is Falsy or uninitialized
                    raise ValueError()
            except (KeyError, ValueError):  # catches if mnemonic is Falsy or doesn't exist in dict tree
                new_machine_wallet = self.default_wallet.create(self.provider)[1]
                composer_yaml['services']['payments']['environment']['TWO1_WALLET_MNEMONIC'] = new_machine_wallet
                new_wallet = new_machine_wallet

        return 0, new_wallet

    def list_services(self):
        """ List available services to sell.
        """
        service_image_data = requests.get(os.path.join(
            Two1Composer.DOCKERHUB_API_URL, Two1Composer.DOCKERHUB_REPO, 'tags')).json().get('results')
        valid_service_names = set([image_data['name'].split('service-')[1] for image_data in
                                   service_image_data if re.match(r'^service-', image_data['name'])])
        return list(valid_service_names)

    def pull_latest_images(self, images):
        """ Pull latest images from 21 DockerHub.

        Args:
            images (list): List of images to pull from the 21 DockerHub.
        """
        for image_tag in images:
            self.docker_client.pull(Two1Composer.DOCKERHUB_REPO, image_tag, stream=False)
        return 0

    def start_services(self, services, failed_to_start_hook, started_hook, failed_to_restart_hook, restarted_hook,
                       failed_to_up_hook, up_hook):
        """ Start selected services.

        Args:
            services (list): List of services to start.
            failed_to_start_hook (Callable): A callable hook that takes in a service name and is run when said service
                                             fails to start.
            started_hook (Callable): A callable hook that takes in a service name and is run when said service starts.
            failed_to_restart_hook (Callable): A callable hook that takes in a service name and is run when said service
                                               fails to restart.
            restarted_hook (Callable): A callable hook that takes in a service name and is run when said service
                                       restarts.
            failed_to_up_hook (Callable): A callable hook that takes in a service name and is run when said service
                                          fails to go up.
            up_hook (Callable): A callable hook that takes in a service name and is run when said service goes up.

        Returns:
            dict: Dictionary with service as key and value as dictionary.
                  Inner dictionary has format {"started": bool, "message": str, "order": int}.

        Raises:

        """
        self._start_sell_service('base', failed_to_start_hook, started_hook, failed_to_up_hook, up_hook)
        self._start_sell_service('router', failed_to_start_hook, started_hook, failed_to_up_hook, up_hook)
        self._start_sell_service('payments', failed_to_start_hook, started_hook, failed_to_up_hook, up_hook)

        self._restart_sell_service('router', failed_to_start_hook, started_hook, failed_to_restart_hook, restarted_hook,
                                   failed_to_up_hook, up_hook)

        # Attempt to start all market services
        for service_name in services:
            # create nginx routes for service_name
            self._create_service_route(service_name)
            # add service_name to docker compose file
            with self.ComposerYAMLContext() as docker_compose_yaml:
                username = docker_compose_yaml['services']['payments']['environment']['TWO1_USERNAME']
                password = docker_compose_yaml['services']['payments']['environment']['TWO1_PASSWORD']
                mnemonic = docker_compose_yaml['services']['payments']['environment']['TWO1_WALLET_MNEMONIC']
                docker_compose_yaml['services'][service_name] = {
                    'image': '%s:%s' % (Two1Composer.DOCKERHUB_REPO, 'service-' + service_name),
                    'container_name': 'sell_%s' % service_name,
                    'depends_on': ['base'],
                    'restart': 'always',
                    'environment': {
                        "TWO1_USERNAME": str(username),
                        "TWO1_PASSWORD": str(password),
                        "TWO1_WALLET_MNEMONIC": str(mnemonic),
                        "SERVICE": str(service_name),
                        "PAYMENT_SERVER_IP": "http://%s:%s" % (self.machine_host, self.machine_config["server_port"])
                    },
                    'volumes': [
                        Two1Composer.DB_DIR + ":/usr/src/db/"
                    ],
                    'logging': {
                        'driver': 'json-file'
                    },
                    'cap_drop': [
                        'ALL'
                    ],
                    'cap_add': [
                        'DAC_OVERRIDE',
                        'NET_RAW',
                    ],
                }
                link_str = '%s:%s' % (service_name, service_name)
                if link_str not in docker_compose_yaml['services']['router']['links']:
                    docker_compose_yaml['services']['router']['links'].append(link_str)

            # attempt to build service_name
            self._start_sell_service(service_name, failed_to_start_hook, started_hook, failed_to_up_hook, up_hook)

        self._restart_sell_service('router', failed_to_start_hook, started_hook, failed_to_restart_hook, restarted_hook,
                                   failed_to_up_hook, up_hook)

    def _start_sell_service(self, service_name, failed_to_start_hook, started_hook, failed_to_up_hook, up_hook,
                            timeout=Two1Composer.SERVICE_START_TIMEOUT):
        try:
            subprocess.check_output(["docker-compose", "-f", Two1Composer.COMPOSE_FILE, "up", "-d", service_name],
                                    stderr=subprocess.DEVNULL, env=self.machine_env)
        except subprocess.CalledProcessError:
            failed_to_start_hook(service_name)
        else:
            started_hook(service_name)
            if service_name == 'router':
                time.sleep(5)
            elif service_name != 'router' and service_name != 'base':
                start = time.clock()

                exec_id = self.docker_client.exec_create('sell_router', "curl %s:5000" % service_name)['Id']
                self.docker_client.exec_start(exec_id)
                running = True

                while time.clock() - start < timeout and running is True:
                    running = self.docker_client.exec_inspect(exec_id)['Running']

                if running is True:
                    failed_to_up_hook(service_name)
                else:
                    up_hook(service_name)

    def _restart_sell_service(self, service_name, failed_to_start_hook, started_hook, failed_to_restart_hook,
                              restarted_hook, failed_to_up_hook, up_hook):
        try:
            self.docker_client.stop("sell_%s" % service_name)
        except:
            is_restart = False
        else:
            is_restart = True

        self._start_sell_service(service_name, failed_to_restart_hook if is_restart else failed_to_start_hook,
                                 restarted_hook if is_restart else started_hook, failed_to_up_hook, up_hook)

    def stop_services(self, services,
                      service_found_stopped_and_removed_hook,
                      service_failed_to_stop_hook,
                      service_failed_to_be_removed_hook,
                      service_not_found_hook):
        """ Stop selected services and remove containers.

        Args:
            services (list): List of services to stop.
            service_found_stopped_and_removed_hook (Callable): A callable hook that takes in a service name and is run
                                                               when said service is found, stopped, and removed.
            service_failed_to_stop_hook (Callable): A callable hook that takes in a service name and is run when said
                                                    service fails to be stopped.
            service_failed_to_be_removed_hook (Callable): A callable hook that takes in a service name and is run when
                                                          said service fails to be removed.
            service_not_found_hook (Callable): A callable hook that takes in a service name and is run when said service
                                               isn't found.

        """
        running_container_names = self.docker_client.containers(filters={"status": "running"})
        for container_name in running_container_names:
            running_service_name = list(self.names_from_containers([container_name]))[0]
            if running_service_name in services:
                try:
                    self.docker_client.stop(container_name)
                except:
                    service_failed_to_stop_hook(running_service_name)
                else:  # container stopped
                    try:
                        self.docker_client.remove_container(container_name)
                    except:
                        service_failed_to_be_removed_hook(running_service_name)
                    else:  # container
                        service_found_stopped_and_removed_hook(running_service_name)

    def silently_force_stop_all_services(self):
        running_container_names = self.docker_client.containers(filters={"status": "running"})
        for container_name in running_container_names:
            self.docker_client.remove_container(container_name, force=True)

    @staticmethod
    def names_from_containers(containers):
        """ Return names from containers.

        Args:
            containers (list): List of containers as returned by self.docker_client.containers
        """
        return frozenset([service['Names'][0][6:] for service in containers])

    def status_services(self, services,
                        service_nonexistent_hook,
                        service_running_hook,
                        service_exited_hook,
                        service_unknown_state_hook):
        """ Gets running status of specified services.

        Args:
            services (list): List of services to get status for.
            service_nonexistent_hook (Callable): A callable hook that takes in a service name and is run when said
                                                 service is non-existent.
            service_running_hook (Callable): A callable hook that takes in a service name and is run when said service
                                             is running.
            service_exited_hook (Callable): A callable hook that takes in a service name and is run when said service
                                            has exited.
            service_unknown_state_hook (Callable): A callable hook that takes in a service name and is run when said
                                                   service is in an unknown state.
        Returns:
            dict: Dictionary with service as key and value as dictionary.
            Inner dictionary has format: {"status": str, "message": str}.
            "Status" choices are: Not found, Running, Exited, Unable to contact.
        """

        existent_services = self.names_from_containers(self.docker_client.containers(all=True))
        running_services = self.names_from_containers(self.docker_client.containers(filters={"status": "running"}))
        exited_services = self.names_from_containers(self.docker_client.containers(filters={"status": "exited"}))

        for service_name in services:
            if service_name in running_services:
                service_running_hook(service_name)
            elif service_name in exited_services:
                service_exited_hook(service_name)
            elif service_name in existent_services:
                service_unknown_state_hook(service_name)
            else:
                service_nonexistent_hook(service_name)

    def get_running_services(self):
        """ Get list of running services.

        Returns:
            (list) started services
        """
        return list(set(self.names_from_containers(self.docker_client.containers(
            filters={"status": "running"}))).difference(set(Two1Composer.BASE_SERVICES)))

    def status_router(self, service_running_hook, service_unknown_state_hook):
        """ Get status of Nginx router container.

        Args:
            service_running_hook (Callable): A callable hook that takes in a service name and is run when said service
                                             is running.
            service_unknown_state_hook (Callable): A callable hook that takes in a service name and is run when said
                                                   service is in an unknown state.
        """
        if len(self.docker_client.containers(all=True, filters={"name": "sell_router", "status": "running"})) == 1:
            service_running_hook("router")
        else:
            service_unknown_state_hook("router")

    def status_payments_server(self, service_running_hook, service_unknown_state_hook):
        """ Get status of payment channels server.

        Args:
            service_running_hook (Callable): A callable hook that takes in a service name and is run when said service
                                             is running.
            service_unknown_state_hook (Callable): A callable hook that takes in a service name and is run when said
                                                   service is in an unknown state.
        """
        if len(self.docker_client.containers(all=True, filters={"name": "sell_payments", "status": "running"})) == 1:
            service_running_hook("payments")
        else:
            service_unknown_state_hook("payments")

    @staticmethod
    def _create_base_server(server_port):
        """ Create nginx base server config.

        Args:
            server_port (int): port for 21 sell server.
        """
        try:
            # create nginx router dirs
            shutil.rmtree(Two1Composer.SITES_ENABLED_PATH, ignore_errors=True)
            shutil.rmtree(Two1Composer.SITES_AVAILABLE_PATH, ignore_errors=True)
            os.makedirs(Two1Composer.SITES_ENABLED_PATH, exist_ok=True)
            os.makedirs(Two1Composer.SITES_AVAILABLE_PATH, exist_ok=True)

            # create base nginx server
            with open(os.path.join(Two1Composer.SITES_ENABLED_PATH,
                                   "two1baseserver"), 'w') as f:
                f.write("server {\n"
                        "    listen " + str(server_port) + ";\n"
                        "    include /etc/nginx/sites-available/*;\n"
                        "}\n"
                        )
        except Exception:
            raise exceptions.Two1ComposerServiceDefinitionException()

    @staticmethod
    def _create_service_route(service):
        """ Create route for container service.
        """
        os.makedirs(Two1Composer.SITES_AVAILABLE_PATH, exist_ok=True)
        try:
            with open(os.path.join(Two1Composer.SITES_AVAILABLE_PATH, service), 'w') as f:
                f.write("location /" + service + " {\n"
                        "    rewrite ^/" + service + "(.*) /$1 break;\n"
                        "    proxy_pass http://" + service + ":" + str(5000) + ";\n"
                        "    proxy_set_header Host $host;\n"
                        "    proxy_set_header X-Real-IP $remote_addr;\n"
                        "    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"
                        "}\n")
        except Exception:
            raise exceptions.Two1ComposerRouteException()

    @staticmethod
    def _create_payments_route():
        """ Add route to payments server.
        """
        os.makedirs(Two1Composer.SITES_AVAILABLE_PATH, exist_ok=True)
        try:
            # write nginx route for payments server
            with open(os.path.join(Two1Composer.SITES_AVAILABLE_PATH, "payments"), 'w') as f:
                f.write("location /payment {\n"
                        "    proxy_pass http://payments:" + str(5000) + ";\n"
                        "    proxy_set_header Host $host;\n"
                        "    proxy_set_header X-Real-IP $remote_addr;\n"
                        "    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"
                        "}\n")
        except Exception:
            raise exceptions.Two1ComposerRouteException()

    def publish_service(self, service_name, rest_client, published_hook, already_published_hook, failed_to_publish_hook,
                        unknown_publish_error_hook):
        strm, stat = self.docker_client.get_archive('sell_%s' % service_name, '/usr/src/app/manifest.yaml')

        with tarfile.open(fileobj=BytesIO(strm.read()), mode='r') as tf:
            manifest = yaml.load(tf.extractfile(stat[u'name']).read().decode())

        try:
            resp = rest_client.publish({"manifest": manifest,
                                        "marketplace": "21mkt"})
        except ServerRequestError as e:
            if e.status_code == 403 and e.data.get("error") == "TO600":
                already_published_hook(service_name)
            else:
                failed_to_publish_hook(service_name)
        except:
            unknown_publish_error_hook(service_name)
        else:
            if resp.status_code == 201:
                published_hook(service_name)
            else:
                failed_to_publish_hook(service_name)

    def read_server_config(self):
        try:
            with open(Two1Composer.COMPOSE_FILE) as f:
                return yaml.load(f)

        except FileNotFoundError:
            return {}

    def get_services_mnemonic(self):
        if os.path.isfile(Two1Composer.COMPOSE_FILE):
            with self.ComposerYAMLContext() as composer_yaml:
                try:
                    maybe_mnemonic = composer_yaml['services']['payments']['environment']['TWO1_WALLET_MNEMONIC']
                except KeyError:
                    rv = None
                else:
                    rv = maybe_mnemonic
        else:
            rv = None
        return rv
Esempio n. 4
0
def take_screenshot(url, width, height, screenshot_path, html_path, pdf_path):
    docker = Client(base_url="unix://run/docker.sock")

    container = docker.create_container(
        image="themyarchive_chrome",
        # http://csmarosi.github.io/sigbus.html
        volumes=["/dev/shm", "/run/shm"],
        host_config=docker.create_host_config(binds=[
            "/dev/shm:/dev/shm",
            "/run/shm:/run/shm",
        ])
    )["Id"]
    docker.start(container)

    try:
        run_command = functools.partial(docker_container_run_command, docker, container)

        docker.put_archive(container, "/home", open(app.config["CHROME_PATH"]))
        run_command(["chown", "-R", "chrome:chrome", "/home/chrome"])

        run_command(["/usr/bin/Xvfb", ":0", "-screen", "0", "%dx%dx24" % (width, height)],
                    user="******", detach=True)
        time.sleep(5)

        run_command(["/bin/bash", "-c", "DISPLAY=:0 openbox-session"],
                    user="******", detach=True)
        time.sleep(5)

        run_command(["/bin/bash", "-c", "DISPLAY=:0 google-chrome --no-sandbox %s" % pipes.quote(url).encode("utf-8")],
                    user="******", detach=True)
        time.sleep(120)

        xdotool = lambda cmds: [(run_command(["/bin/bash", "-c", "DISPLAY=:0 xdotool %s" % cmd], user="******"),
                                 time.sleep(1))
                                for cmd in cmds]
        xdotool(["mousemove %d %d" % (width - 48, 43),
                 "click 1"])
        time.sleep(60)

        xdotool(["keydown ctrl",
                 "key s",
                 "keyup ctrl",
                 "key p",
                 "key a",
                 "key g",
                 "key e",
                 "key period",
                 "key p",
                 "key n",
                 "key g",
                 "key Return"])
        time.sleep(5)

        xdotool(["keydown ctrl",
                 "key w",
                 "keyup ctrl"])

        xdotool(["keydown ctrl",
                 "key s",
                 "keyup ctrl",
                 "key p",
                 "key a",
                 "key g",
                 "key e",
                 "key period",
                 "key h",
                 "key t",
                 "key m",
                 "key l",
                 "key Return"])
        time.sleep(60)

        xdotool(["keydown ctrl",
                 "key p",
                 "keyup ctrl",
                 "key Return",
                 "keydown shift",
                 "key d",
                 "keyup shift",
                 "key o",
                 "key w",
                 "key n",
                 "key l",
                 "key o",
                 "key a",
                 "key d",
                 "key s",
                 "key slash",
                 "key p",
                 "key a",
                 "key g",
                 "key e",
                 "key period",
                 "key p",
                 "key d",
                 "key f",
                 "key Return"])
        time.sleep(10)

        with tempfile.NamedTemporaryFile(suffix=".tar") as f:
            f.write(docker.get_archive(container, "/home/chrome/Downloads")[0].read())
            f.flush()

            tmp_dir = tempfile.mkdtemp()
            try:
                subprocess.check_call(["tar", "xf", f.name, "-C", tmp_dir])
                downloads = os.path.join(tmp_dir, "Downloads")
                files = os.listdir(downloads)
                logger.info("Files: %r", files)

                features = []

                screenshot = filter(lambda name: name.endswith(".png"), files)[0]
                shutil.move(os.path.join(downloads, screenshot), screenshot_path)
                files.remove(screenshot)

                pdf = filter(lambda name: name.endswith(".pdf"), files)[0]
                shutil.move(os.path.join(downloads, pdf), pdf_path)
                files.remove(pdf)

                try:
                    data_dir = filter(lambda name: os.path.isdir(os.path.join(downloads, name)), files)[0]
                except IndexError:
                    pass
                else:
                    shutil.move(os.path.join(downloads, data_dir), html_path)
                    files.remove(data_dir)

                if len(files):
                    shutil.move(os.path.join(downloads, files[0]), os.path.join(html_path, "index.html"))
                    features.append("html")

                os.chmod(screenshot_path, 0644)
                os.chmod(html_path, 0755)
                for root, dirs, files in os.walk(html_path):
                    for d in dirs:
                        os.chmod(os.path.join(root, d), 0755)
                    for fl in files:
                        os.chmod(os.path.join(root, fl), 0644)
            finally:
                shutil.rmtree(tmp_dir)
    finally:
        docker.remove_container(container, force=True)

    return {"features": features}
Esempio n. 5
0
class Pgdocker(Postgres):
    DB_TYPE = "pgdocker"

    def __init__(self, db_config):
        super(Pgdocker, self).__init__(db_config)

        self.image = db_config.pgdocker_image
        self.docker = Client("unix:///var/run/docker.sock", timeout=1800)
        self.backup_path = os.path.join(db_config.backup_dir, 'default.tar')

        # пытаемся прочесть конфиг оставшийся с последнего запуска
        self._container_file = 'pgdocker_db'
        try:
            with open(self._container_file, 'rt') as f:
                container_name, port = f.read().split(' ')
        except (FileNotFoundError, ValueError):
            # это новый контейнер
            log.info('New container')
            container_name = ''
            port = random.randint(40000, 50000)

        self._container_name = db_config.container or container_name
        self.port = str(db_config.port or port)

        try:
            self._start(self._container_name)
        except (NotFound, NullResource):
            # NotFound - этого контейнера больше нет на этом хосте
            # NullResource - база была удалена
            log.warning('Database container not found, you can try create new')

    def _save_container(self, new_id):
        # Если это было рандомное имя, то запоминаем его, отсекаю слеш в начале
        if new_id:
            self._container_name = self.docker.inspect_container(
                new_id)["Name"][1:]
        else:
            self._container_name = ''
        with open(self._container_file, 'wt') as f:
            f.write(' '.join((self._container_name, self.port)))

    def _create_container(self):
        log.debug('create container. name=%s, port=%s', self._container_name,
                  self.port)
        return self.docker.create_container(
            image=self.image,
            name=self._container_name,
            detach=True,
            ports=[5432],
            host_config=self.docker.create_host_config(
                port_bindings={5432: self.port}),
            environment={'POSTGRES_PASSWORD': self.password},
        )['Id']

    def _start(self, container):
        log.debug('try start db container')
        self.docker.start(container)
        # ждем около 30 секунд пока сервер начнет слушать на порту и инициализаует файловую систему
        # Сразу после поднятия psql: FATAL:  the database system is starting up
        for i in range(0, 15):
            try:
                super(Pgdocker, self)._run_console_command(['psql', '--list'],
                                                           5)
                return

            except (ConnectionRefusedError, RuntimeError,
                    subprocess.TimeoutExpired):
                time.sleep(2)

        raise TimeoutError('Pgdocker was not started')

    def _remove(self, container):
        log.debug('remove %s', container)
        # Контейнер не остановлен, используем флаг force
        self.docker.remove_container(container, v=True, force=True)

    def create(self):
        log.info('Create container with postgres_db. Name %s, port %s',
                 self._container_name, self.port)
        new_container_id = self._create_container()
        try:
            self._start(new_container_id)
            super(Pgdocker, self).create()
            # параметры id контейнера меняются только если был успешный старт
            self._save_container(new_container_id)
        except Exception as e:
            self._remove(new_container_id)
            raise e

    def drop(self):
        log.info('Drop pgdocker container')
        self._remove(self._container_name)
        self._save_container(None)

    def backup(self):
        log.info('Backup database container %s on server %s',
                 self._container_name, self.addr)
        # https://www.postgresql.org/docs/9.4/static/backup-file.html
        # The database server must be shut down in order to get a usable backup
        self.docker.stop(self._container_name, timeout=60)
        self.docker.wait(self._container_name)
        with open(self.backup_path, "wb") as f:
            (stream,
             stat) = self.docker.get_archive(self._container_name,
                                             "/var/lib/postgresql/data/.")
            buffer = stream.read(10000000)
            while buffer:
                f.write(buffer)
                buffer = stream.read(10000000)
        self._start(self._container_name)

    def restore(self):
        # Если это tar архив, то пробуем развернуть его как filesystem backup в остальных случаях пытаемся
        # обработать его как стандартный архив постгреса
        command = ['file', self.backup_path]
        if subprocess.check_output(command, timeout=self.quick_operation_timeout).decode(). \
                find('POSIX tar archive') != -1:
            log.info('Restore filesystem backup for container %s on server %s',
                     self._container_name, self.addr)
            # Сначала сдедует почистить текущие файлы базы данных, для этого удаляем контейнер вместе с томом бд
            # Кроме того, при копировании бэкапа права установятся в root но видимо перепишутся при первом запуске контейнера
            self._remove(self._container_name)
            new_container_id = self._create_container()
            with open(self.backup_path, "rb") as f:
                self.docker.put_archive(new_container_id,
                                        "/var/lib/postgresql/data", f)
            self._start(new_container_id)
            self._save_container(new_container_id)
        else:
            super(Pgdocker, self).restore()
Esempio n. 6
0
class Two1ComposerContainers(Two1Composer):
    """ Manage machine-payable microservices in containers.
    """
    def __init__(self):
        self._connected = ComposerState.DISCONNECTED
        self.provider = TwentyOneProvider()
        self.default_wallet = Two1Wallet(self.wallet_file, self.provider)

    class ServiceManager:
        """ Query and modify user services persisting at cls.USER_SERVICES_FILE.
        """

        USER_SERVICES_FILE = os.path.join(Two1Composer.BASE_DIR,
                                          "user-services.json")

        class Image(namedtuple('Image', 'docker_hub_account repository tag')):
            def _asdict(self):
                # Fixes a bug for Python 3.4 users
                # https://bugs.python.org/issue24931
                'A new OrderedDict mapping field names to their values'
                return OrderedDict(zip(self._fields, self))

            @property
            def is_dockerhub_image(self):
                """ Returns: True iff Image instance has all fields.
                """
                return self.docker_hub_account and self.repository and self.tag

            @property
            def is_local_image(self):
                """ Returns: True iff Image instance doesn't have docker_hub_account but has all other fields.
                """
                return not self.docker_hub_account and self.repository and self.tag

            def __str__(self):
                """ Returns: Docker image name constructed from Image instance fields.
                """
                if self.is_dockerhub_image:
                    return '%s/%s:%s' % (self.docker_hub_account,
                                         self.repository, self.tag)
                elif self.is_local_image:
                    return '%s:%s' % (self.repository, self.tag)
                else:
                    raise ValueError()

            @classmethod
            def from_string(cls, image_name):
                """ Constructs an Image instance from a docker image name.

                Args:
                    image_name (str): A docker image name.

                Returns:
                    Image: An Image instance.
                """
                slashes = re.findall('/', image_name)
                colons = re.findall(':', image_name)

                if len(slashes) == 1:
                    if len(colons) == 1 and image_name.find(
                            '/') < image_name.find(':'):
                        docker_hub_account, rest = image_name.split('/')
                        repository, tag = rest.split(':')
                        return cls(docker_hub_account=docker_hub_account,
                                   repository=repository,
                                   tag=tag)
                    elif len(colons) == 0:
                        docker_hub_account, repository = image_name.split('/')
                        return cls(docker_hub_account=docker_hub_account,
                                   repository=repository,
                                   tag='latest')
                elif len(slashes) == 0:
                    if len(colons) == 1:
                        repository, tag = image_name.split(':')
                        return cls(docker_hub_account=None,
                                   repository=repository,
                                   tag=tag)
                    elif len(colons) == 0:
                        return cls(docker_hub_account=None,
                                   repository=image_name,
                                   tag='latest')
                raise ValueError()

        @classmethod
        def get_image(cls, service_name):
            """ Constructs an Image instance for a service.

            Args:
                service_name (str): The name of either a 21 service in the 21dotco/two1 repository or a user service
                                    added to ServiceManager.USER_SERVICES_FILE by ServiceManager.add_service.

            Returns:
                Image: An Image instance corresponding to the given service.
            """
            if service_name in cls.available_21_services():
                return cls.Image(
                    docker_hub_account='21dotco',
                    repository='two1',
                    tag=service_name if service_name
                    in Two1Composer.BASE_SERVICES else 'service-%s' %
                    service_name)
            elif service_name in cls.available_user_services():
                return cls.Image(**cls._get_user_service_dict()[service_name])
            else:
                raise ValueError()

        @classmethod
        def available_services(cls):
            """ Returns: All available service names.
            """
            return cls.available_21_services() | cls.available_user_services()

        @classmethod
        def available_21_services(cls):
            """ Returns: All available 21 services by querying Docker Hub.
            """
            service_image_data = requests.get(
                os.path.join(Two1Composer.DOCKERHUB_API_URL,
                             Two1Composer.DOCKERHUB_REPO,
                             'tags')).json().get('results')
            return set([
                image_data['name'].split('service-')[1]
                for image_data in service_image_data
                if re.match(r'^service-', image_data['name'])
            ])

        @classmethod
        def available_user_services(cls):
            """ Returns: All available user services.
            """
            return set(cls._get_user_service_dict().keys())

        @classmethod
        def add_service(cls, service_name, image_name_string,
                        service_successfully_added_hook,
                        service_already_exists_hook,
                        service_failed_to_add_hook):
            """ Adds a new service definition to ServiceManager.USER_SERVICES_FILE.

            Args:
                service_name (str): Name of the service definition to add.
                image_name_string (str): Docker image name for the service definition.
            """
            service_dict = cls._get_user_service_dict()
            if service_name in service_dict:
                service_already_exists_hook(service_name)
            else:
                service_dict[service_name] = cls.Image.from_string(
                    image_name_string)._asdict()
                if cls._commit_user_service_dict(service_dict):
                    service_successfully_added_hook(service_name)
                else:
                    service_failed_to_add_hook(service_name)

        @classmethod
        def remove_service(cls, service_name,
                           service_successfully_removed_hook,
                           service_does_not_exists_hook,
                           service_failed_to_remove_hook):
            """ Removes a service definition from ServiceManager.USER_SERVICES_FILE.

            Args:
                service_name (str): Name of the service definition to remove.
            """
            service_dict = cls._get_user_service_dict()
            if service_name in service_dict:
                del service_dict[service_name]
                if cls._commit_user_service_dict(service_dict):
                    service_successfully_removed_hook(service_name)
                else:
                    service_failed_to_remove_hook(service_name)
            else:
                service_does_not_exists_hook(service_name)

        @classmethod
        def _get_user_service_dict(cls):
            """ Returns: ServiceManager.USER_SERVICES_FILE as a dict.
            """
            try:
                with open(cls.USER_SERVICES_FILE, 'r') as data_file:
                    service_dict = json.load(data_file)
            except:
                return {}
            else:
                return service_dict

        @classmethod
        def _commit_user_service_dict(cls, service_dict):
            """ Writes a dict of user services to ServiceManager.USER_SERVICES_FILE in json format.

            Args:
                service_dict (dict): A dictionary of user services of the form
                                     {service_name : _as_dict representation of corresponding Image instance..}.

            Returns:
                bool: True iff no exceptions were raised when writing service_dict to ServiceManager.USER_SERVICES_FILE
                      as json.
            """
            try:
                with open(cls.USER_SERVICES_FILE, 'w') as outfile:
                    json.dump(service_dict, outfile)
            except:
                return False
            else:
                return True

    class ComposerYAMLContext(YamlDataContext):
        """ Context manager for composer YAML service file.
        """
        def __init__(self,
                     username=None,
                     password=None,
                     server_port=None,
                     mnemonic=None):
            self.username = username
            self.password = password
            self.server_port = server_port
            self.mnemonic = mnemonic
            super().__init__(Two1Composer.COMPOSE_FILE)

        def __enter__(self):
            sup = super().__enter__()
            for service in self.data['services']:
                service_definition = self.data['services'][service]
                if 'environment' in service_definition:

                    if 'TWO1_USERNAME' in service_definition[
                            'environment'] and self.username is not None:
                        service_definition['environment'][
                            'TWO1_USERNAME'] = self.username

                    if 'TWO1_PASSWORD' in service_definition[
                            'environment'] and self.password is not None:
                        service_definition['environment'][
                            'TWO1_PASSWORD'] = self.password

                    if 'TWO1_WALLET_MNEMONIC' in service_definition[
                            'environment'] and self.mnemonic is not None:
                        service_definition['environment'][
                            'TWO1_WALLET_MNEMONIC'] = self.mnemonic

                    if 'PAYMENT_SERVER_IP' in service_definition[
                            'environment'] and self.server_port is not None:
                        rest = service_definition['environment'][
                            'PAYMENT_SERVER_IP'].rsplit(':', maxsplit=1)[-1]
                        service_definition['environment'][
                            'PAYMENT_SERVER_IP'] = '%s:%s' % (rest,
                                                              self.server_port)
            return sup

        def _filler(self):
            """ Create the base service description file.
            """
            return {
                'version': '2',
                'services': {
                    'base': {
                        'image': '%s:base' % Two1Composer.DOCKERHUB_REPO,
                    },
                    'router': {
                        'image':
                        '%s:router' % Two1Composer.DOCKERHUB_REPO,
                        'container_name':
                        'sell_router',
                        'restart':
                        'always',
                        'volumes': [
                            Two1Composer.SITES_ENABLED_PATH +
                            ":/etc/nginx/sites-enabled",
                            Two1Composer.SITES_AVAILABLE_PATH +
                            ":/etc/nginx/sites-available",
                        ],
                        'ports':
                        ['%s:%s' % (self.server_port, self.server_port)],
                        'links': [
                            'payments:payments',
                        ],
                    },
                    'payments': {
                        'image': '%s:payments' % Two1Composer.DOCKERHUB_REPO,
                        'depends_on': ['base'],
                        'container_name': 'sell_payments',
                        'restart': 'always',
                        'environment': {
                            "TWO1_USERNAME": str(self.username),
                            "TWO1_PASSWORD": str(self.password),
                            "TWO1_WALLET_MNEMONIC": str(self.mnemonic)
                        },
                        'volumes': [Two1Composer.DB_DIR + ":/usr/src/db/"],
                        'logging': {
                            'driver': 'json-file'
                        },
                        'cap_drop': ['ALL'],
                        'cap_add': [
                            'DAC_OVERRIDE',
                            'NET_RAW',
                        ],
                    }
                }
            }

    # public api
    def connect(self, machine_env, host, machine_config_file):
        """ Connect service composer to machine layer.

        Args:
            machine_env (dict): Environment dictionary for the docker client of the machine layer.
            host (str): Hostname of the machine layer docker daemon.
            machine_config_file (str): Path to the config file for the machine layer.
        """
        self.machine_env = machine_env
        self.machine_host = host
        with open(machine_config_file, 'r') as f:
            self.machine_config = json.load(f)
        self.docker_client = Client(
            **docker_env(assert_hostname=False, environment=self.machine_env))
        self._connected = ComposerState.CONNECTED

    def initialize_server(self, username, password, server_port, wallet=None):
        """ Initialize micropayments server.

        Define boilerplate services, networks, and volumes composer file
        and nginx server config.

        Generates a wallet mnemonic if non-existent.

        Args:
            username (str): Username to log in with.
            password (str): Password to log in with.
            server_port (int): The server port that the router is running on.
            wallet: The wallet to use for the payments server and subsequent services.
        """
        self._create_base_server(
            server_port)  # create base router server config
        self._create_payments_route()  # create route to payments server

        new_wallet = None  # rv[1], not None if mnemonic is replaced in this function

        # generate service description (yaml)
        with self.ComposerYAMLContext(username, password,
                                      server_port) as composer_yaml:
            try:
                mnemonic = composer_yaml['services']['payments'][
                    'environment']['TWO1_WALLET_MNEMONIC']
                if not mnemonic or mnemonic == str(
                        None):  # if mnemonic is Falsy or uninitialized
                    raise ValueError()
            except (
                    KeyError, ValueError
            ):  # catches if mnemonic is Falsy or doesn't exist in dict tree
                new_machine_wallet = self.default_wallet.create(
                    self.provider)[1]
                composer_yaml['services']['payments']['environment'][
                    'TWO1_WALLET_MNEMONIC'] = new_machine_wallet
                new_wallet = new_machine_wallet

        return 0, new_wallet

    def pull_image(self, image, image_sucessfully_pulled_hook,
                   image_failed_to_pull_hook, image_is_local_hook,
                   image_is_malformed_hook):
        """ Pulls an Image instance iff it is a Docker Hub image.

        Args:
            image (Image): An Image instance.
        """
        if image.is_dockerhub_image:
            try:
                self.docker_client.pull(
                    '%s/%s' % (image.docker_hub_account, image.repository),
                    tag=image.tag,
                    stream=False)
            except:
                image_failed_to_pull_hook(image)
            else:
                image_sucessfully_pulled_hook(image)
        elif image.is_local_image:
            image_is_local_hook(image)
        else:
            image_is_malformed_hook(image)

    def start_services(self, service_names, failed_to_start_hook, started_hook,
                       failed_to_restart_hook, restarted_hook,
                       failed_to_up_hook, up_hook):
        """ Start selected services.

        Args:
            service_names (list): List of service names to start.
            failed_to_start_hook (Callable): A callable hook that takes in a service name and is run when said service
                                             fails to start.
            started_hook (Callable): A callable hook that takes in a service name and is run when said service starts.
            failed_to_restart_hook (Callable): A callable hook that takes in a service name and is run when said service
                                               fails to restart.
            restarted_hook (Callable): A callable hook that takes in a service name and is run when said service
                                       restarts.
            failed_to_up_hook (Callable): A callable hook that takes in a service name and is run when said service
                                          fails to go up.
            up_hook (Callable): A callable hook that takes in a service name and is run when said service goes up.
        """
        self._start_sell_service('base', failed_to_start_hook, started_hook,
                                 failed_to_up_hook, up_hook)
        self._start_sell_service('router', failed_to_start_hook, started_hook,
                                 failed_to_up_hook, up_hook)
        self._start_sell_service('payments', failed_to_start_hook,
                                 started_hook, failed_to_up_hook, up_hook)

        self._restart_sell_service('router', failed_to_start_hook,
                                   started_hook, failed_to_restart_hook,
                                   restarted_hook, failed_to_up_hook, up_hook)

        # Attempt to start all market services
        for service_name in service_names:
            try:
                image = self.ServiceManager.get_image(service_name)
                container_name = self.service_name_2_container_name(
                    service_name)

                # create nginx routes for service_name
                self._create_service_route(service_name)
                # add service_name to docker compose file
                with self.ComposerYAMLContext() as docker_compose_yaml:
                    username = docker_compose_yaml['services']['payments'][
                        'environment']['TWO1_USERNAME']
                    password = docker_compose_yaml['services']['payments'][
                        'environment']['TWO1_PASSWORD']
                    mnemonic = docker_compose_yaml['services']['payments'][
                        'environment']['TWO1_WALLET_MNEMONIC']
                    docker_compose_yaml['services'][service_name] = {
                        'image': str(image),
                        'container_name': container_name,
                        'depends_on': ['base'],
                        'restart': 'always',
                        'environment': {
                            "TWO1_USERNAME":
                            str(username),
                            "TWO1_PASSWORD":
                            str(password),
                            "TWO1_WALLET_MNEMONIC":
                            str(mnemonic),
                            "SERVICE":
                            str(service_name),
                            "PAYMENT_SERVER_IP":
                            "http://%s:%s" %
                            (self.machine_host,
                             self.machine_config["server_port"])
                        },
                        'volumes': [Two1Composer.DB_DIR + ":/usr/src/db/"],
                        'logging': {
                            'driver': 'json-file'
                        },
                        'cap_drop': ['ALL'],
                        'cap_add': [
                            'DAC_OVERRIDE',
                            'NET_RAW',
                        ],
                    }
                    link_str = '%s:%s' % (service_name, service_name)
                    if link_str not in docker_compose_yaml['services'][
                            'router']['links']:
                        docker_compose_yaml['services']['router'][
                            'links'].append(link_str)
            except:
                # something went wrong while configuring service_name
                failed_to_start_hook(service_name)
            else:
                # attempt to build service_name
                self._start_sell_service(service_name, failed_to_start_hook,
                                         started_hook, failed_to_up_hook,
                                         up_hook)

        self._restart_sell_service('router', failed_to_start_hook,
                                   started_hook, failed_to_restart_hook,
                                   restarted_hook, failed_to_up_hook, up_hook)

    def _start_sell_service(self,
                            service_name,
                            failed_to_start_hook,
                            started_hook,
                            failed_to_up_hook,
                            up_hook,
                            timeout=Two1Composer.SERVICE_START_TIMEOUT):
        try:
            subprocess.check_output([
                "docker-compose", "-f", Two1Composer.COMPOSE_FILE, "up", "-d",
                service_name
            ],
                                    stderr=subprocess.DEVNULL,
                                    env=self.machine_env)
        except subprocess.CalledProcessError:
            failed_to_start_hook(service_name)
        else:
            started_hook(service_name)
            if service_name == 'router':
                time.sleep(5)
            elif service_name != 'router' and service_name != 'base':
                start = time.clock()

                exec_id = self.docker_client.exec_create(
                    'sell_router', "curl %s:5000" % service_name)['Id']
                self.docker_client.exec_start(exec_id)
                running = True

                while time.clock() - start < timeout and running is True:
                    running = self.docker_client.exec_inspect(
                        exec_id)['Running']

                if running is True:
                    failed_to_up_hook(service_name)
                else:
                    up_hook(service_name)

    def _restart_sell_service(self, service_name, failed_to_start_hook,
                              started_hook, failed_to_restart_hook,
                              restarted_hook, failed_to_up_hook, up_hook):
        try:
            self.docker_client.stop("sell_%s" % service_name)
        except:
            is_restart = False
        else:
            is_restart = True

        self._start_sell_service(
            service_name,
            failed_to_restart_hook if is_restart else failed_to_start_hook,
            restarted_hook if is_restart else started_hook, failed_to_up_hook,
            up_hook)

    def stop_services(self, service_names,
                      service_found_stopped_and_removed_hook,
                      service_failed_to_stop_hook,
                      service_failed_to_be_removed_hook,
                      service_not_found_hook):
        """ Stop selected services and remove containers.

        Args:
            service_names (set): Set of services to stop.
            service_found_stopped_and_removed_hook (Callable): A callable hook that takes in a service name and is run
                                                               when said service is found, stopped, and removed.
            service_failed_to_stop_hook (Callable): A callable hook that takes in a service name and is run when said
                                                    service fails to be stopped.
            service_failed_to_be_removed_hook (Callable): A callable hook that takes in a service name and is run when
                                                          said service fails to be removed.
            service_not_found_hook (Callable): A callable hook that takes in a service name and is run when said service
                                               isn't found.

        """
        for service_name in service_names:
            if service_name in self.get_running_services():
                container_name = self.service_name_2_container_name(
                    service_name)
                try:
                    self.docker_client.stop(container_name)
                except:
                    service_failed_to_stop_hook(service_name)
                else:
                    try:
                        self.docker_client.remove_container(container_name)
                    except:
                        service_failed_to_be_removed_hook(service_name)
                    else:
                        service_found_stopped_and_removed_hook(service_name)
            else:
                service_not_found_hook(service_name)

    def silently_force_stop_all_services(self):
        running_container_names = self.docker_client.containers(
            filters={"status": "running"})
        for container_name in running_container_names:
            self.docker_client.remove_container(container_name, force=True)

    @staticmethod
    def container_names_2_service_names(container_definitions):
        """ Return service names from container definitions.

        See service_name_2_container_name for the inverse operation but on one service name.

        Args:
            container_definitions (list): List of container descriptions as returned by self.docker_client.containers.

        Returns:
            set: Set of service names generated by removing the 'sell_' prefix from the containers' names.
        """
        return set([
            container_definition['Names'][0][6:]
            for container_definition in container_definitions
        ])

    @staticmethod
    def service_name_2_container_name(service_name):
        """ Generates a container name from a service name by prepending 'sell_'
        """
        return 'sell_%s' % service_name

    def status_services(self, services):
        """ Gets running status of specified services.

        Args:
            services (list): List of services to get status for.
        """

        existent_services = self.get_services(all=True)
        running_services = self.get_services(filters={"status": "running"})
        exited_services = self.get_services(filters={"status": "exited"})

        return {
            "running": running_services & services,
            "exited": exited_services & services,
            "nonexistent": services - existent_services
        }

    def get_services(self, *args, **kwargs):
        """ Call docker_client.containers | convert resulting container names to service names | remove base services
        """
        return self.container_names_2_service_names(
            self.docker_client.containers(*args, **
                                          kwargs)) - Two1Composer.BASE_SERVICES

    def get_running_services(self):
        """ Get list of currently running services that aren't 21 base services.

        Returns:
            set: Set of currently running services.
        """
        return self.get_services(filters={"status": "running"})

    def status_router(self, service_running_hook, service_unknown_state_hook):
        """ Get status of Nginx router container.

        Args:
            service_running_hook (Callable): A callable hook that takes in a service name and is run when said service
                                             is running.
            service_unknown_state_hook (Callable): A callable hook that takes in a service name and is run when said
                                                   service is in an unknown state.
        """
        if len(
                self.docker_client.containers(all=True,
                                              filters={
                                                  "name": "sell_router",
                                                  "status": "running"
                                              })) == 1:
            service_running_hook("router")
        else:
            service_unknown_state_hook("router")

    def status_payments_server(self, service_running_hook,
                               service_unknown_state_hook):
        """ Get status of payment channels server.

        Args:
            service_running_hook (Callable): A callable hook that takes in a service name and is run when said service
                                             is running.
            service_unknown_state_hook (Callable): A callable hook that takes in a service name and is run when said
                                                   service is in an unknown state.
        """
        if len(
                self.docker_client.containers(all=True,
                                              filters={
                                                  "name": "sell_payments",
                                                  "status": "running"
                                              })) == 1:
            service_running_hook("payments")
        else:
            service_unknown_state_hook("payments")

    @staticmethod
    def _create_base_server(server_port):
        """ Create nginx base server config.

        Args:
            server_port (int): port for 21 sell server.
        """
        try:
            # create nginx router dirs
            shutil.rmtree(Two1Composer.SITES_ENABLED_PATH, ignore_errors=True)
            shutil.rmtree(Two1Composer.SITES_AVAILABLE_PATH,
                          ignore_errors=True)
            os.makedirs(Two1Composer.SITES_ENABLED_PATH, exist_ok=True)
            os.makedirs(Two1Composer.SITES_AVAILABLE_PATH, exist_ok=True)

            # create base nginx server
            with open(
                    os.path.join(Two1Composer.SITES_ENABLED_PATH,
                                 "two1baseserver"), 'w') as f:
                f.write("server {\n"
                        "    listen " + str(server_port) + ";\n"
                        "    include /etc/nginx/sites-available/*;\n"
                        "}\n")
        except Exception:
            raise exceptions.Two1ComposerServiceDefinitionException()

    @staticmethod
    def _create_service_route(service):
        """ Create route for container service.
        """
        os.makedirs(Two1Composer.SITES_AVAILABLE_PATH, exist_ok=True)
        try:
            with open(os.path.join(Two1Composer.SITES_AVAILABLE_PATH, service),
                      'w') as f:
                f.write(
                    "location /" + service + " {\n"
                    "    rewrite ^/" + service + "/?(.*) /$1 break;\n"
                    "    proxy_pass http://" + service + ":" + str(5000) +
                    ";\n"
                    "    proxy_set_header Host $host;\n"
                    "    proxy_set_header X-Real-IP $remote_addr;\n"
                    "    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"
                    "}\n")
        except Exception:
            raise exceptions.Two1ComposerRouteException()

    @staticmethod
    def _create_payments_route():
        """ Add route to payments server.
        """
        os.makedirs(Two1Composer.SITES_AVAILABLE_PATH, exist_ok=True)
        try:
            # write nginx route for payments server
            with open(
                    os.path.join(Two1Composer.SITES_AVAILABLE_PATH,
                                 "payments"), 'w') as f:
                f.write(
                    "location /payment {\n"
                    "    proxy_pass http://payments:" + str(5000) + ";\n"
                    "    proxy_set_header Host $host;\n"
                    "    proxy_set_header X-Real-IP $remote_addr;\n"
                    "    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"
                    "}\n")
        except Exception:
            raise exceptions.Two1ComposerRouteException()

    def publish_service(self, service_name, host_override, rest_client,
                        published_hook, already_published_hook,
                        failed_to_publish_hook, unknown_publish_error_hook):
        strm, stat = self.docker_client.get_archive(
            'sell_%s' % service_name, '/usr/src/app/manifest.yaml')

        with tarfile.open(fileobj=BytesIO(strm.read()), mode='r') as tf:
            manifest = yaml.load(tf.extractfile(stat[u'name']).read().decode())
        manifest['host'] = host_override

        try:
            resp = rest_client.publish({
                "manifest": manifest,
                "marketplace": "21mkt"
            })
        except ServerRequestError as e:
            if e.status_code == 403 and e.data.get("error") == "TO600":
                already_published_hook(service_name)
            else:
                failed_to_publish_hook(service_name)
        except:
            unknown_publish_error_hook(service_name)
        else:
            if resp.status_code == 201:
                published_hook(service_name)
            else:
                failed_to_publish_hook(service_name)

    def read_server_config(self):
        try:
            with open(Two1Composer.COMPOSE_FILE) as f:
                return yaml.load(f)

        except FileNotFoundError:
            return {}

    def get_services_mnemonic(self):
        if os.path.isfile(Two1Composer.COMPOSE_FILE):
            with self.ComposerYAMLContext() as composer_yaml:
                try:
                    maybe_mnemonic = composer_yaml['services']['payments'][
                        'environment']['TWO1_WALLET_MNEMONIC']
                except KeyError:
                    rv = None
                else:
                    rv = maybe_mnemonic
        else:
            rv = None
        return rv
Esempio n. 7
0
    key_container = client.create_container("johncclayton/syncthing-cli", "/generate-identities.sh",
                                            environment={
                                                "KEY_HOSTS": " ".join(HOSTS.keys())
                                            })
    print("Generating keys...")
    key_response = client.start(container=key_container.get('Id'))
    client.wait(container=key_container.get('Id'))

    # now copy the files from within the container to the local disk
    if os.path.exists(key_pathname):
        shutil.rmtree(key_pathname)

    os.mkdir(key_pathname)

    print("Copying key data from container...")
    strm, stat = client.get_archive(key_container, "/keys")
    data = io.BytesIO(strm.read())
    tarobj = tarfile.open(fileobj=data)
    tarobj.extractall(path=os.getcwd())

# find the syncthing config.xml template and populate it with all the known device Ids.  This
# file is then used within all the syncthing images to ensure that all hosts know about each other.
env = Environment(loader=FileSystemLoader(os.path.join(os.getcwd(), "templates")))
config_template = env.get_template('config_xml.tpl')

devices = []

print("Creating config.xml for each device...")

# create a single config.xml file that has the ID values of every known key
for subdir, dirs, files in os.walk(key_pathname):