示例#1
0
    async def delete_containers(self):
        '''
        Deletes the containers in docker and the db records
        '''
        logging.info('Deleting containers')

        client = aiodocker.Docker()

        containers = await client.containers.list(all=True)
        # Remove the containers first
        for container in Container.select():
            logging.debug('Deleting %s', container.name)
            for item in containers:
                if item.id == container.container_id:
                    await item.delete(v=True, force=True)

        # Purge the db. Need to find a more efficient way
        # to do this
        for container in Container.select():
            container.delete_instance()

        for port in Port.select():
            port.delete_instance()

        await client.close()
def app_delete_container(host, cid):
    try:
        node = Node.find(host)
    except ValueError:
        return Response("{} is not in the cluster".format(host), status=422)

    Container.find(node, cid).delete()
    return Response("", status=204)
示例#3
0
 def setUpClass(cls):
     logging.debug('Clearing the db...')
     Container.objects.all().delete()
     logging.debug('Creating user...')
     cls.user = User.objects.get_or_create(username='******')[0]
     logging.debug('Adding to the DB two bundles...')
     cls.ids = [Container.create('search_test_1', examples.bundles1(), cls.user, False).id,
                Container.create('search_test_1_other', examples.bundles2(), cls.user, False).id]
示例#4
0
    def get_available_address(self, count):
        # Add three to the number of containers to account for the gateway,
        # network, broadcast address
        subnet = utility.Net.generate_cidr(self.config.network.address,
                                           count + 3)

        ip = netaddr.IPNetwork(subnet)

        # Calc the gateway as network address + 1
        gateway = utility.Net.ipint_to_str(ip.first + 1)

        # Broadcast is the last
        broadcast = utility.Net.ipint_to_str(ip.last)

        addresses = [
            str(address) for address in list(netaddr.IPSet([
                subnet,
            ]))
        ]

        if gateway in addresses:
            addresses.remove(gateway)
        if broadcast in addresses:
            addresses.remove(broadcast)
        if self.config.network.address in addresses:
            addresses.remove(self.config.network.address)

        # Remove existing addresses
        for container in Container.select(Container.ip):
            addresses.remove(container.ip)

        return addresses, subnet, gateway
示例#5
0
    def build_container_map(self):
        '''
        Creates dictionaries for fast retrieval of container information by index.
        '''

        container_map_by_port = {}
        container_map_by_ip = {}
        container_map_by_name = {}

        containers = Container.select()
        for container in containers:
            container_map_by_name[container.name] = container

        end_points = Port.select(Port.number, Port.protocol, Container.ip, Container.name,
                                 Container.container_id, Container.start_delay, Container.start_retry_count,
                                 Container.start_on_create, Container.sub_domain) \
            .join(Container)\
            .order_by(Port.number)

        for end_point in end_points:
            port_key = self.get_port_map_key(end_point.number,
                                             end_point.protocol)
            container = end_point.container

            if port_key not in container_map_by_port:
                container_map_by_port[port_key] = {}

            ip = utility.Net.ipstr_to_int(container.ip)
            container_map_by_port[port_key][ip] = container

            if ip not in container_map_by_ip:
                container_map_by_ip[ip] = container

        return container_map_by_port, container_map_by_ip, container_map_by_name
def app_node_container_status(host, cid):
    try:
        node = Node.find(host)
    except ValueError:
        return Response("{} is not in the cluster".format(node), status=422)
    container = Container.find(node, cid)
    return json.dumps(container.status())
示例#7
0
def converge(key='*'):
    # note that this gets the machines in the `key` cluster
    machines = Machine.load_files(key=key) # machine name => machine

    # note that this gets all containers accross all users.
    # we are converging the entire `key` machine group within the cluster
    containers = Container.load_files(machines) # deploy id => container

    print machines, containers

    create_threads = []
    destroy_threads = []
    
    for m in machines.values():
        target_containers = [c for c in containers.values() if c.machine == m]
        (to_create, to_delete, t1, t2) = m.converge(target_containers)
        create_threads.append(t1)
        destroy_threads.append(t2)

    print "Starting container creation"
    for t in create_threads:
        t.start()

    for t in create_threads:
        t.join()
    print "Finished container creation"

    print "Starting container destruction"
    for t in destroy_threads:
        t.start()

    for t in destroy_threads:
        t.join()
    print "Finished container destruction"
示例#8
0
 async def stop_all_containers(self):
     containers = Container.select()
     client = await self.client_pool.get()
     for container in containers:
         try:
             client.stop(container=container.name)
         except:
             pass
示例#9
0
def sleep(key='alpha-dev'):
    machines = Machine.load_files(key=key) # machine name => machine
    print machines
    containers = Container.load_files(machines, key='*') # deploy id => container

    for m in machines.values():
        target_containers = [c for c in containers.values() if c.machine == m and c.d_id not in ['zenrez','adshare']]
        m.bulk_sleep(target_containers)
示例#10
0
 def view(self, container_map_by_port=None, container_map_by_ip=None):
     '''
     Displays the created containers specified by the current configuration
     '''
     containers = Container.select()
     for container in containers:
         print('{}.{}.{}\t\t{}'.format(container.name, container.sub_domain,
                                       self.config.network.domain,
                                       container.ip))
示例#11
0
 def create_container(cls, engines):
     for engine in engines:
         response = requests.get(engine.url('/containers'))
         results = response.json()['results']
         for result in results:
             container = Container()
             container.uid = result['Id']
             container.image = Image.objects.get(uid=result['Image'])
             container.name = result['Name']
             container.engine = engine
             container.save()
示例#12
0
def get_context(file_path: str) -> Context:
    with open(file_path, 'r') as file:
        context_dict = json.load(file)

        context = Context(**context_dict)

        for i in range(len(context.containers)):
            container = context.containers[i] = Container(**context.containers[i])
            container.ports = Port(**container.ports)

        for i in range(len(context.networks)):
            context.networks[i] = Network(**context.networks[i])

        return context
def app_migrate_container(host, cid):
    try:
        node = Node.find(host)
    except ValueError:
        return Response("{} is not in the cluster".format(host), status=422)

    container = Container.find(node, cid)

    nodes = Node.all()

    strategy = AllocationStrategy.from_name(current_app.config['strategy'])
    selected_node = strategy.select_node(nodes, container.service())

    new_container = container.migrate(selected_node)
    return Response(json.dumps(
        {"Started": new_container,
         "Stopped": container}, cls=ContainerJSONEncoder), status=201)
def app_new_container():
    try:
        service = request.form['service']
    except KeyError:
        return Response("service field should be provided", status=422)
    nodes = Node.all()

    strategy = AllocationStrategy.from_name(current_app.config['strategy'])
    selected_node = strategy.select_node(nodes, service)

    try:
        image = request.form['image']
    except KeyError:
        image = "soulou/msc-thesis-fibo-http-service"

    started_container = Container.create(selected_node, service, image)
    return Response(json.dumps(started_container, cls=ContainerJSONEncoder), status=201)
示例#15
0
def container_crud():
    if request.method == 'POST':
        new_warehouse_id = request.form['warehouse_id']
        new_container_number = request.form['container_number']
        try:
            new_container = Container(
                warehouse_id=new_warehouse_id,
                container_number=new_container_number).create()
            return redirect('/container_crud')
        except:
            return redirect('/container_crud')
            # return 'There was an issue adding your container'

    else:
        containers = get_all_containers()
        warehouses = get_all_warehouses()
        return render_template('containers.html',
                               containers=containers,
                               warehouses=warehouses)
示例#16
0
 def save(self, owner, commit=True):
     if self.errors:
         raise ValueError("The %s could not be %s because the data didn't"
                      " validate." % ('Container', 'created'))
         
     container = Container.create(self.cleaned_data['rec_id'], self.bundle, owner, self.cleaned_data['public'])
     save = False
     
     if 'submission' in self.files:
         file_sub = self.files['submission']
         sub = Submission.objects.create()
         sub.content.save(sub.timestamp.strftime('%Y-%m-%d%H-%M-%S')+file_sub._name, file_sub)
         container.submission = sub
         save = True
         
     for l in self.cleaned_data['license']:
         container.license.add(l)
         save = True
         
     if save:
         container.save()
     return container
示例#17
0
    def generate_host_names(self, naming_cfg, count):

        containers = Container.select(Container.name)

        unique_hosts = {}

        with open(naming_cfg.word_file, "r") as f:
            array = []
            for line in f:
                word = re.sub(naming_cfg.allowable_host_chars, "",
                              line.lower())
                word_len = len(word)
                if word_len >= naming_cfg.min_host_len and \
                                word_len <= naming_cfg.max_host_len:
                    array.append(word)

            while len(unique_hosts) < count:
                index = random.randint(1, len(array))
                name = array[index - 1]
                if not name in unique_hosts and name not in containers:
                    unique_hosts[name] = name

            return list(unique_hosts)
示例#18
0
    async def create_containers(self, update):
        # Clean up all the containers
        if not update:
            await self.delete_containers()

        # Make sure the defined images are available and config the ports
        count, image_ports, unique_ports = await self.setup_images(
            self.config.images, update)

        addresses, subnet, gateway = self.get_available_address(
            self.config.network.hosts)

        # Do not recreate the network during an update. docker does not provide a
        # way to update the network so during an update it would need to be dropped and
        # updated. The network id for any existing containers would need to be updated
        if not update:
            # Create a network for the containers
            await self.create_network(self.config.network.hosts, subnet,
                                      gateway)

            # Apply the iptable rules required for the configured images
            rules = IPTableRules(self.config)
            rules.create(subnet, unique_ports)

        logging.info('Creating containers')

        # Create a unique list of host names
        naming = Naming()
        host_names = naming.generate_host_names(self.config.naming, count)

        scripts_dir = ''.join([os.getcwd(), SCRIPTS_DIR])
        logging.info('Using %s for script directory', scripts_dir)

        client = aiodocker.Docker()
        for image in self.config.images:

            # If the image has no exposed ports then there is no use in
            # creating a container
            if image.name not in image_ports:
                continue

            ports = [port.text for port in image_ports[image.name]]
            ports = dict.fromkeys(ports, {})
            for count in range(image.count):
                host_name = host_names.pop()
                ip = addresses.pop(0)
                mac = utility.Net.generate_mac(ip)
                config = {
                    'Hostname': host_name,
                    'Image': image.name,
                    'ExposedPorts': ports,
                    'MacAddress': mac,
                    'Env': image.env_variables,
                    'NetworkingConfig': {
                        'EndpointsConfig': {
                            'clab': {
                                'IPAMConfig': {
                                    'IPv4Address': ip,
                                },
                            }
                        }
                    }
                }

                if not image.startup_script is None and len(
                        image.startup_script) > 0:
                    config['HostConfig'] = {
                        'Binds': ['{}:{}'.format(scripts_dir, SCRIPTS_DIR)],
                    }

                logging.debug('Creating container %s:%s', host_name,
                              image.name)
                container = await client.containers.create(config,
                                                           name=host_name)

                # Persist the container info to the db with the ports
                # Ports are used to determine what listeners to create
                new_container = Container.create(
                    container_id=container.id,
                    name=host_name,
                    ip=ip,
                    mac=mac,
                    start_delay=image.start_delay,
                    start_retry_count=image.start_retry_count,
                    start_on_create=image.start_on_create,
                    sub_domain=image.sub_domain)

                # Some containers perform a lot of startup work when run for the first time
                # To mitigate this containers can be started and stopped on creation
                if image.start_on_create:
                    await self.start_container_on_create(
                        image, new_container, client)

                for port in image_ports[image.name]:
                    Port.create(container=new_container.id,
                                number=port.num,
                                protocol=port.protocol)

        await client.close()
示例#19
0
    def testOAuthAccess(self):
        self.user = User.objects.create_user('jane', '*****@*****.**', 'toto')
        self.resource = Resource.objects.get_or_create(name='api', url='/api/')
        self.CONSUMER_KEY = 'dpf43f3p2l4k3l03'
        self.CONSUMER_SECRET = 'kd94hf93k423kf44'
        self.consumer, _ = Consumer.objects.get_or_create(
            key=self.CONSUMER_KEY, secret=self.CONSUMER_SECRET,
            defaults={
                'name': 'Test',
                'description': 'Testing...'
        })
        pb = ProvBundle()
        pb._decode_JSON_container('')
        self.bundle = Container.create('test_bundle', pb, self.user)
        
        c = Client()
        response = c.get("/oauth/request_token/")
        self.assertEqual(response.status_code, 401)
        import time
        parameters = {
            'oauth_consumer_key': self.CONSUMER_KEY,
            'oauth_signature_method': 'PLAINTEXT',
            'oauth_signature': '%s&' % self.CONSUMER_SECRET,
            'oauth_timestamp': str(int(time.time())),
            'oauth_nonce': 'requestnonce',
            'oauth_version': '1.0',
            'oauth_callback': 'http://test/request_token_ready',
            'scope': 'api',
            }
        response = c.get("/oauth/request_token/", parameters)
        self.assertEqual(response.status_code, 200)
        token = list(Token.objects.all())[-1]
        self.assertIn(token.key, response.content)
        self.assertIn(token.secret, response.content)
        self.assertTrue(token.callback_confirmed)

        parameters = {'oauth_token': token.key,}
        response = c.get("/oauth/authorize/", parameters)
        self.assertEqual(response.status_code, 302)
        self.assertIn(token.key, response['Location'])
        c.login(username='******', password='******')
        self.assertFalse(token.is_approved)
        response = c.get("/oauth/authorize/", parameters)
        self.assertEqual(response.status_code, 200)
        
        # fake authorization by the user
        parameters['authorize_access'] = 1
        response = c.post("/oauth/authorize/", parameters)
        self.assertEqual(response.status_code, 302)
        token = Token.objects.get(key=token.key)
        self.assertIn(token.key, response['Location'])
        self.assertTrue(token.is_approved)
        c.logout()
        
        # Exchange the Request token for an Access token
        parameters = {
            'oauth_consumer_key': self.CONSUMER_KEY,
            'oauth_token': token.key,
            'oauth_signature_method': 'PLAINTEXT',
            'oauth_signature': '%s&%s' % (self.CONSUMER_SECRET, token.secret),
            'oauth_timestamp': str(int(time.time())),
            'oauth_nonce': 'accessnonce',
            'oauth_version': '1.0',
            'oauth_verifier': token.verifier,
            'scope': 'api',
            }
        response = c.get("/oauth/access_token/", parameters)
        self.assertEqual(response.status_code, 200)
        access_token = list(Token.objects.filter(token_type=Token.ACCESS))[-1]
        self.assertIn(access_token.key, response.content)
        self.assertIn(access_token.secret, response.content)
        self.assertEqual(access_token.user.username, self.user.username)
        
        # Generating signature base string
        parameters = {
            'oauth_consumer_key': self.CONSUMER_KEY,
            'oauth_token': access_token.key,
            'oauth_signature_method': 'HMAC-SHA1',
            'oauth_timestamp': str(int(time.time())),
            'oauth_nonce': 'accessresourcenonce',
            'oauth_version': '1.0',
        }
        url_path = "/api/v0/bundle/%d/" % self.bundle.id
        oauth_request = oauth.Request.from_token_and_callback(access_token, http_url='http://testserver' + url_path, parameters=parameters)
        signature_method = oauth.SignatureMethod_HMAC_SHA1()
        signature = signature_method.sign(oauth_request, self.consumer, access_token)
        parameters['oauth_signature'] = signature
        response = c.get(url_path + '?format=json', parameters)
        self.assertEqual(response.status_code, 200)
示例#20
0
 def _save_container(self, dao, container, run_id):
     session = dao.get_session()
     c = Container(docker_id=container.get('Id'), run_id=run_id)
     session.add(c)
     session.commit()
     return c
def restore():

    # Set flags
    utils.set_flags(sys.argv)

    # Load config
    config = Path(__file__).parent / "config.yml"
    with open(config) as file:
        # Load config
        config_list = yaml.full_load(file)
        log = Log(config_list['log']['log_dir'])
        containers = Container.instantiate_containers(config_list)

    # If any container names were passed as parameters, do only restore them
    containers_wanted = {name: container for name, container in containers.items() if name in sys.argv}
    if containers_wanted:
        containers = containers_wanted
    # If no container was chosen ask for it
    elif not utils.all_containers:
        containers_to_choose_from = [container.name for container in containers.values()]
        terminal_menu = TerminalMenu(containers_to_choose_from, title="For which Nextcloud instance do you want "
                                                                      "to restore a backup?")
        choice_index = terminal_menu.show()
        containers = {containers_to_choose_from[choice_index]: containers.get(containers_to_choose_from[choice_index])}

    container: Container
    for container in containers.values():

        # Start restore
        _print("----------------------------------------------")
        _print(F"Restore backup for {container.name}")

        backup_dir = os.scandir(container.backup_dir)
        backup_files = {file.name: file for file in backup_dir if
                        file.is_file() and file.name.startswith(container.name) and file.name.endswith(".tar.gz")}
        if len(backup_files) < 1:
            _print(F"{Fore.YELLOW}No backups found for {container.name}{Style.RESET_ALL}")
            break
        backup_files_to_choose_from = [file.name for file in backup_files.values()]
        backup_files_to_choose_from.sort(reverse=True)
        _print()

        # Choose backup to restore from
        terminal_menu = TerminalMenu(backup_files_to_choose_from, title="Which backup do you want to restore?")
        choice_index = terminal_menu.show()
        backup_file = backup_files.get(backup_files_to_choose_from[choice_index])
        print(backup_file.path)

        # Confirm restore
        if not utils.no_confirm:
            confirm = input(F"Are you sure that you want to restore {backup_files_to_choose_from[choice_index]}? "
                            F"(Type: yes)\n").lower() == "yes"
        else:
            confirm = False

        # Do the restore
        if confirm or utils.no_confirm:
            result = container.restore_backup(backup_file.path)
        else:
            break

        # Print result and log
        if result:
            _print(F"{Fore.GREEN}Backup {container.restore_tar_file} for {container.name} successfully restored.{Style.RESET_ALL}")
            if not utils.no_log and config_list['log']['logging']:
                log.log(F"Restore backup ; {container.name} ; {container.restore_tar_file_path} ; SUCCESS")
        else:
            _print(F"{Fore.RED}Could not restore {container.restore_tar_file} for {container.name}.{Style.RESET_ALL}")
            for func, traceback in container.exceptions.items():
                _print()
                _print(F"{Fore.YELLOW}Exception occurred in method: Container.{func}(){Style.RESET_ALL}")
                _print(traceback)
                _print()
            if not utils.no_log and config_list['log']['logging']:
                log.log(F"Restore backup ; {container.name} ; {container.restore_tar_file_path} ; FAIL")
def upgrade():

    # Set flags
    utils.set_flags(sys.argv)

    # Load config
    config = Path(__file__).parent / "config.yml"
    with open(config) as file:
        # Load config
        config_list = yaml.full_load(file)
        log = Log(config_list['log']['log_dir'])
        containers = Container.instantiate_containers(config_list)

    # If any container names were passed as parameters, do only upgrade them
    containers_wanted = {
        name: container
        for name, container in containers.items() if name in sys.argv
    }
    if containers_wanted:
        containers = containers_wanted

    # If no container was chosen ask for it
    elif not utils.all_containers:
        containers_to_choose_from = [
            container.name for container in containers.values()
        ]
        terminal_menu = TerminalMenu(
            containers_to_choose_from,
            title="Which Nextcloud instance do you want to "
            "upgrade?")
        choice_index = terminal_menu.show()
        containers = {
            containers_to_choose_from[choice_index]:
            containers.get(containers_to_choose_from[choice_index])
        }

    # Loop through Nextcloud container instances
    container: Container
    for container in containers.values():
        go_on = True
        # Make a backup
        if not utils.no_backup:
            utils.keep_maintenance_mode = True
            go_on = backup.backup()
        if go_on:
            # Make the upgrade
            utils.keep_maintenance_mode = True if "--maintenance" in sys.argv else False
            _print("----------------------------------------------")
            _print(
                F"Start upgrade for {container.name} at {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
            )
            result = container.upgrade()
            if result == 1:
                _print(
                    F"{Fore.GREEN}{container.name} upgraded successfully{Style.RESET_ALL}"
                )
                upgrade_status = True
            elif result == 2:
                _print(
                    F"{Fore.GREEN}No upgrades available for {container.name}.{Style.RESET_ALL}"
                )
                upgrade_status = True
            else:
                _print(
                    F"{Fore.RED}Upgrade for {container.name} failed{Style.RESET_ALL}"
                )
                for func, traceback in container.exceptions.items():
                    _print()
                    _print(
                        F"{Fore.YELLOW}Exception occurred in method: Container.{func}(){Style.RESET_ALL}"
                    )
                    _print(traceback)
                    _print()
            # Log upgrade
            if not utils.no_log and config_list['log']['logging']:
                if upgrade_status:
                    log.log(F"Upgrade ; {container.name} ; SUCCESS")
                else:
                    log.log(F"Upgrade ; {container.name} ; FAIL")
                    if len(log.exceptions) > 0:
                        for func, traceback in log.exceptions.items():
                            _print()
                            _print(
                                F"{Fore.YELLOW}Exception occurred in method: Log.{func}(){Style.RESET_ALL}"
                            )
                            _print(traceback)
                            _print()
示例#23
0
文件: views.py 项目: kerberis/Kremlin
def install_docker(request, domain, password):
    container = Container.create_new_container(domain, password)
    return HttpResponse(u"Youpi, ça a marché ! %s" % container.ip)
示例#24
0
def list():
    containers = [
        Container(container) for container in cli.containers(all=True)
    ]
    return containers_list('containers/list.html', containers)
示例#25
0
from database import Base, engine, Session
from models import Node, Container, ContainerNodeAssoc

Base.metadata.create_all(engine)

session = Session()

n1 = Container(parent_id=None, data="container1")
n2 = Container(parent_id=None, data="container2")

c1 = Container(data="child1")
c2 = Container(data="child2")
c3 = Container(data="child3")
c4 = Container(data="child4")
c5 = Container(data="child5")
c6 = Container(data="child6")

node1 = Node(name="node1")
node2 = Node(name="node2")
node3 = Node(name="node3")
node4 = Node(name="node4")
node5 = Node(name="node5")
node6 = Node(name="node6")

# populate the assoc table
# node1 in container1
node1_container1 = ContainerNodeAssoc(container=c1, node=node1)
node2_container2 = ContainerNodeAssoc(container=c2, node=node2)
node3_container3 = ContainerNodeAssoc(container=c3, node=node3)
node4_container4 = ContainerNodeAssoc(container=c4, node=node4)
node5_container5 = ContainerNodeAssoc(container=c5, node=node5)
示例#26
0
文件: api.py 项目: YilinHao/w3-prov
    def obj_create(self, bundle, request=None, **kwargs):
        '''
        Function to create a bundle via the API.
        Arguments (in JSON) are as follows:
            rec_id - the name for the Bundle (REQUIRED)
            content - the content in JSON format (or URL REQUIRDE)
            URL - a URL reference to the JSON content of the bundle * (OR content REQUIRED)
            public - indicator whether the Bundle to be made public, False if not provided
            licenses - a list of names corresponding to License objects in the DB
            submission - a file submitted which represents the Bundle **
        
        * - The URL is only opened and it's content parsed if and only if 'content' is empty
        ** - The file content is not parsed at all, it is just saved for later usage and
             if file is submitted the HTTP call should be in 'multipart' encoding format.
        '''
        
        try:
            '''Try to read the content else download and parse the URL file '''
            prov_bundle = ProvBundle()
            if bundle.data['content']:
                prov_bundle._decode_JSON_container(bundle.data['content'])
            else:
                source = urlopen(bundle.data['url'], timeout=5)
                content = source.read()
                source.close()
                prov_bundle._decode_JSON_container(loads(content))
            container = Container.create(bundle.data['rec_id'], prov_bundle, request.user)
        except: 
            raise ImmediateHttpResponse(HttpBadRequest())

        save = False
        if 'public' in bundle.data: 
            container.public = bundle.data['public']
            save = True
            if bundle.data['public']:
                assign('view_container', Group.objects.get(id=PUBLIC_GROUP_ID), container) 
                           
        if 'licenses' in bundle.data:
            for title in bundle.data['licenses']:
                try:
                    lic = License.objects.get(title=title)
                    container.license.add(lic)
                    save = True
                except License.DoesNotExist:
                    pass
        
        if 'submission' in request.FILES:
            file_sub = request.FILES['submission']
            sub = Submission.objects.create()
            sub.content.save(sub.timestamp.strftime('%Y-%m-%d%H-%M-%S')+file_sub._name, file_sub)
            container.submission = sub
            save = True
            
        if 'url' in bundle.data:
            container.url = bundle.data['url']
            save = True
            
        if save:
            container.save()
        bundle.obj = container
        return bundle