async def delete_containers(self):
'''
Deletes the containers in docker and the db records
'''
logging.info('Deleting containers')
client = aiodocker.Docker()
containers = await client.containers.list(all=True)
# Remove the containers first
for container in Container.select():
logging.debug('Deleting %s', container.name)
for item in containers:
if item.id == container.container_id:
await item.delete(v=True, force=True)
# Purge the db. Need to find a more efficient way
# to do this
for container in Container.select():
container.delete_instance()
for port in Port.select():
port.delete_instance()
await client.close()
def app_delete_container(host, cid):
try:
node = Node.find(host)
except ValueError:
return Response("{} is not in the cluster".format(host), status=422)
Container.find(node, cid).delete()
return Response("", status=204)
def setUpClass(cls):
logging.debug('Clearing the db...')
Container.objects.all().delete()
logging.debug('Creating user...')
cls.user = User.objects.get_or_create(username='******')[0]
logging.debug('Adding to the DB two bundles...')
cls.ids = [Container.create('search_test_1', examples.bundles1(), cls.user, False).id,
Container.create('search_test_1_other', examples.bundles2(), cls.user, False).id]
def get_available_address(self, count):
# Add three to the number of containers to account for the gateway,
# network, broadcast address
subnet = utility.Net.generate_cidr(self.config.network.address,
count + 3)
ip = netaddr.IPNetwork(subnet)
# Calc the gateway as network address + 1
gateway = utility.Net.ipint_to_str(ip.first + 1)
# Broadcast is the last
broadcast = utility.Net.ipint_to_str(ip.last)
addresses = [
str(address) for address in list(netaddr.IPSet([
subnet,
]))
]
if gateway in addresses:
addresses.remove(gateway)
if broadcast in addresses:
addresses.remove(broadcast)
if self.config.network.address in addresses:
addresses.remove(self.config.network.address)
# Remove existing addresses
for container in Container.select(Container.ip):
addresses.remove(container.ip)
return addresses, subnet, gateway
def build_container_map(self):
'''
Creates dictionaries for fast retrieval of container information by index.
'''
container_map_by_port = {}
container_map_by_ip = {}
container_map_by_name = {}
containers = Container.select()
for container in containers:
container_map_by_name[container.name] = container
end_points = Port.select(Port.number, Port.protocol, Container.ip, Container.name,
Container.container_id, Container.start_delay, Container.start_retry_count,
Container.start_on_create, Container.sub_domain) \
.join(Container)\
.order_by(Port.number)
for end_point in end_points:
port_key = self.get_port_map_key(end_point.number,
end_point.protocol)
container = end_point.container
if port_key not in container_map_by_port:
container_map_by_port[port_key] = {}
ip = utility.Net.ipstr_to_int(container.ip)
container_map_by_port[port_key][ip] = container
if ip not in container_map_by_ip:
container_map_by_ip[ip] = container
return container_map_by_port, container_map_by_ip, container_map_by_name
def app_node_container_status(host, cid):
try:
node = Node.find(host)
except ValueError:
return Response("{} is not in the cluster".format(node), status=422)
container = Container.find(node, cid)
return json.dumps(container.status())
def converge(key='*'):
# note that this gets the machines in the `key` cluster
machines = Machine.load_files(key=key) # machine name => machine
# note that this gets all containers accross all users.
# we are converging the entire `key` machine group within the cluster
containers = Container.load_files(machines) # deploy id => container
print machines, containers
create_threads = []
destroy_threads = []
for m in machines.values():
target_containers = [c for c in containers.values() if c.machine == m]
(to_create, to_delete, t1, t2) = m.converge(target_containers)
create_threads.append(t1)
destroy_threads.append(t2)
print "Starting container creation"
for t in create_threads:
t.start()
for t in create_threads:
t.join()
print "Finished container creation"
print "Starting container destruction"
for t in destroy_threads:
t.start()
for t in destroy_threads:
t.join()
print "Finished container destruction"
async def stop_all_containers(self):
containers = Container.select()
client = await self.client_pool.get()
for container in containers:
try:
client.stop(container=container.name)
except:
pass
def sleep(key='alpha-dev'):
machines = Machine.load_files(key=key) # machine name => machine
print machines
containers = Container.load_files(machines, key='*') # deploy id => container
for m in machines.values():
target_containers = [c for c in containers.values() if c.machine == m and c.d_id not in ['zenrez','adshare']]
m.bulk_sleep(target_containers)
def view(self, container_map_by_port=None, container_map_by_ip=None):
'''
Displays the created containers specified by the current configuration
'''
containers = Container.select()
for container in containers:
print('{}.{}.{}\t\t{}'.format(container.name, container.sub_domain,
self.config.network.domain,
container.ip))
def create_container(cls, engines):
for engine in engines:
response = requests.get(engine.url('/containers'))
results = response.json()['results']
for result in results:
container = Container()
container.uid = result['Id']
container.image = Image.objects.get(uid=result['Image'])
container.name = result['Name']
container.engine = engine
container.save()
def get_context(file_path: str) -> Context:
with open(file_path, 'r') as file:
context_dict = json.load(file)
context = Context(**context_dict)
for i in range(len(context.containers)):
container = context.containers[i] = Container(**context.containers[i])
container.ports = Port(**container.ports)
for i in range(len(context.networks)):
context.networks[i] = Network(**context.networks[i])
return context
def app_migrate_container(host, cid):
try:
node = Node.find(host)
except ValueError:
return Response("{} is not in the cluster".format(host), status=422)
container = Container.find(node, cid)
nodes = Node.all()
strategy = AllocationStrategy.from_name(current_app.config['strategy'])
selected_node = strategy.select_node(nodes, container.service())
new_container = container.migrate(selected_node)
return Response(json.dumps(
{"Started": new_container,
"Stopped": container}, cls=ContainerJSONEncoder), status=201)
def app_new_container():
try:
service = request.form['service']
except KeyError:
return Response("service field should be provided", status=422)
nodes = Node.all()
strategy = AllocationStrategy.from_name(current_app.config['strategy'])
selected_node = strategy.select_node(nodes, service)
try:
image = request.form['image']
except KeyError:
image = "soulou/msc-thesis-fibo-http-service"
started_container = Container.create(selected_node, service, image)
return Response(json.dumps(started_container, cls=ContainerJSONEncoder), status=201)
def container_crud():
if request.method == 'POST':
new_warehouse_id = request.form['warehouse_id']
new_container_number = request.form['container_number']
try:
new_container = Container(
warehouse_id=new_warehouse_id,
container_number=new_container_number).create()
return redirect('/container_crud')
except:
return redirect('/container_crud')
# return 'There was an issue adding your container'
else:
containers = get_all_containers()
warehouses = get_all_warehouses()
return render_template('containers.html',
containers=containers,
warehouses=warehouses)
def save(self, owner, commit=True):
if self.errors:
raise ValueError("The %s could not be %s because the data didn't"
" validate." % ('Container', 'created'))
container = Container.create(self.cleaned_data['rec_id'], self.bundle, owner, self.cleaned_data['public'])
save = False
if 'submission' in self.files:
file_sub = self.files['submission']
sub = Submission.objects.create()
sub.content.save(sub.timestamp.strftime('%Y-%m-%d%H-%M-%S')+file_sub._name, file_sub)
container.submission = sub
save = True
for l in self.cleaned_data['license']:
container.license.add(l)
save = True
if save:
container.save()
return container
def generate_host_names(self, naming_cfg, count):
containers = Container.select(Container.name)
unique_hosts = {}
with open(naming_cfg.word_file, "r") as f:
array = []
for line in f:
word = re.sub(naming_cfg.allowable_host_chars, "",
line.lower())
word_len = len(word)
if word_len >= naming_cfg.min_host_len and \
word_len <= naming_cfg.max_host_len:
array.append(word)
while len(unique_hosts) < count:
index = random.randint(1, len(array))
name = array[index - 1]
if not name in unique_hosts and name not in containers:
unique_hosts[name] = name
return list(unique_hosts)
async def create_containers(self, update):
# Clean up all the containers
if not update:
await self.delete_containers()
# Make sure the defined images are available and config the ports
count, image_ports, unique_ports = await self.setup_images(
self.config.images, update)
addresses, subnet, gateway = self.get_available_address(
self.config.network.hosts)
# Do not recreate the network during an update. docker does not provide a
# way to update the network so during an update it would need to be dropped and
# updated. The network id for any existing containers would need to be updated
if not update:
# Create a network for the containers
await self.create_network(self.config.network.hosts, subnet,
gateway)
# Apply the iptable rules required for the configured images
rules = IPTableRules(self.config)
rules.create(subnet, unique_ports)
logging.info('Creating containers')
# Create a unique list of host names
naming = Naming()
host_names = naming.generate_host_names(self.config.naming, count)
scripts_dir = ''.join([os.getcwd(), SCRIPTS_DIR])
logging.info('Using %s for script directory', scripts_dir)
client = aiodocker.Docker()
for image in self.config.images:
# If the image has no exposed ports then there is no use in
# creating a container
if image.name not in image_ports:
continue
ports = [port.text for port in image_ports[image.name]]
ports = dict.fromkeys(ports, {})
for count in range(image.count):
host_name = host_names.pop()
ip = addresses.pop(0)
mac = utility.Net.generate_mac(ip)
config = {
'Hostname': host_name,
'Image': image.name,
'ExposedPorts': ports,
'MacAddress': mac,
'Env': image.env_variables,
'NetworkingConfig': {
'EndpointsConfig': {
'clab': {
'IPAMConfig': {
'IPv4Address': ip,
},
}
}
}
}
if not image.startup_script is None and len(
image.startup_script) > 0:
config['HostConfig'] = {
'Binds': ['{}:{}'.format(scripts_dir, SCRIPTS_DIR)],
}
logging.debug('Creating container %s:%s', host_name,
image.name)
container = await client.containers.create(config,
name=host_name)
# Persist the container info to the db with the ports
# Ports are used to determine what listeners to create
new_container = Container.create(
container_id=container.id,
name=host_name,
ip=ip,
mac=mac,
start_delay=image.start_delay,
start_retry_count=image.start_retry_count,
start_on_create=image.start_on_create,
sub_domain=image.sub_domain)
# Some containers perform a lot of startup work when run for the first time
# To mitigate this containers can be started and stopped on creation
if image.start_on_create:
await self.start_container_on_create(
image, new_container, client)
for port in image_ports[image.name]:
Port.create(container=new_container.id,
number=port.num,
protocol=port.protocol)
await client.close()
def testOAuthAccess(self):
self.user = User.objects.create_user('jane', '*****@*****.**', 'toto')
self.resource = Resource.objects.get_or_create(name='api', url='/api/')
self.CONSUMER_KEY = 'dpf43f3p2l4k3l03'
self.CONSUMER_SECRET = 'kd94hf93k423kf44'
self.consumer, _ = Consumer.objects.get_or_create(
key=self.CONSUMER_KEY, secret=self.CONSUMER_SECRET,
defaults={
'name': 'Test',
'description': 'Testing...'
})
pb = ProvBundle()
pb._decode_JSON_container('')
self.bundle = Container.create('test_bundle', pb, self.user)
c = Client()
response = c.get("/oauth/request_token/")
self.assertEqual(response.status_code, 401)
import time
parameters = {
'oauth_consumer_key': self.CONSUMER_KEY,
'oauth_signature_method': 'PLAINTEXT',
'oauth_signature': '%s&' % self.CONSUMER_SECRET,
'oauth_timestamp': str(int(time.time())),
'oauth_nonce': 'requestnonce',
'oauth_version': '1.0',
'oauth_callback': 'http://test/request_token_ready',
'scope': 'api',
}
response = c.get("/oauth/request_token/", parameters)
self.assertEqual(response.status_code, 200)
token = list(Token.objects.all())[-1]
self.assertIn(token.key, response.content)
self.assertIn(token.secret, response.content)
self.assertTrue(token.callback_confirmed)
parameters = {'oauth_token': token.key,}
response = c.get("/oauth/authorize/", parameters)
self.assertEqual(response.status_code, 302)
self.assertIn(token.key, response['Location'])
c.login(username='******', password='******')
self.assertFalse(token.is_approved)
response = c.get("/oauth/authorize/", parameters)
self.assertEqual(response.status_code, 200)
# fake authorization by the user
parameters['authorize_access'] = 1
response = c.post("/oauth/authorize/", parameters)
self.assertEqual(response.status_code, 302)
token = Token.objects.get(key=token.key)
self.assertIn(token.key, response['Location'])
self.assertTrue(token.is_approved)
c.logout()
# Exchange the Request token for an Access token
parameters = {
'oauth_consumer_key': self.CONSUMER_KEY,
'oauth_token': token.key,
'oauth_signature_method': 'PLAINTEXT',
'oauth_signature': '%s&%s' % (self.CONSUMER_SECRET, token.secret),
'oauth_timestamp': str(int(time.time())),
'oauth_nonce': 'accessnonce',
'oauth_version': '1.0',
'oauth_verifier': token.verifier,
'scope': 'api',
}
response = c.get("/oauth/access_token/", parameters)
self.assertEqual(response.status_code, 200)
access_token = list(Token.objects.filter(token_type=Token.ACCESS))[-1]
self.assertIn(access_token.key, response.content)
self.assertIn(access_token.secret, response.content)
self.assertEqual(access_token.user.username, self.user.username)
# Generating signature base string
parameters = {
'oauth_consumer_key': self.CONSUMER_KEY,
'oauth_token': access_token.key,
'oauth_signature_method': 'HMAC-SHA1',
'oauth_timestamp': str(int(time.time())),
'oauth_nonce': 'accessresourcenonce',
'oauth_version': '1.0',
}
url_path = "/api/v0/bundle/%d/" % self.bundle.id
oauth_request = oauth.Request.from_token_and_callback(access_token, http_url='http://testserver' + url_path, parameters=parameters)
signature_method = oauth.SignatureMethod_HMAC_SHA1()
signature = signature_method.sign(oauth_request, self.consumer, access_token)
parameters['oauth_signature'] = signature
response = c.get(url_path + '?format=json', parameters)
self.assertEqual(response.status_code, 200)
def _save_container(self, dao, container, run_id):
session = dao.get_session()
c = Container(docker_id=container.get('Id'), run_id=run_id)
session.add(c)
session.commit()
return c
def restore():
# Set flags
utils.set_flags(sys.argv)
# Load config
config = Path(__file__).parent / "config.yml"
with open(config) as file:
# Load config
config_list = yaml.full_load(file)
log = Log(config_list['log']['log_dir'])
containers = Container.instantiate_containers(config_list)
# If any container names were passed as parameters, do only restore them
containers_wanted = {name: container for name, container in containers.items() if name in sys.argv}
if containers_wanted:
containers = containers_wanted
# If no container was chosen ask for it
elif not utils.all_containers:
containers_to_choose_from = [container.name for container in containers.values()]
terminal_menu = TerminalMenu(containers_to_choose_from, title="For which Nextcloud instance do you want "
"to restore a backup?")
choice_index = terminal_menu.show()
containers = {containers_to_choose_from[choice_index]: containers.get(containers_to_choose_from[choice_index])}
container: Container
for container in containers.values():
# Start restore
_print("----------------------------------------------")
_print(F"Restore backup for {container.name}")
backup_dir = os.scandir(container.backup_dir)
backup_files = {file.name: file for file in backup_dir if
file.is_file() and file.name.startswith(container.name) and file.name.endswith(".tar.gz")}
if len(backup_files) < 1:
_print(F"{Fore.YELLOW}No backups found for {container.name}{Style.RESET_ALL}")
break
backup_files_to_choose_from = [file.name for file in backup_files.values()]
backup_files_to_choose_from.sort(reverse=True)
_print()
# Choose backup to restore from
terminal_menu = TerminalMenu(backup_files_to_choose_from, title="Which backup do you want to restore?")
choice_index = terminal_menu.show()
backup_file = backup_files.get(backup_files_to_choose_from[choice_index])
print(backup_file.path)
# Confirm restore
if not utils.no_confirm:
confirm = input(F"Are you sure that you want to restore {backup_files_to_choose_from[choice_index]}? "
F"(Type: yes)\n").lower() == "yes"
else:
confirm = False
# Do the restore
if confirm or utils.no_confirm:
result = container.restore_backup(backup_file.path)
else:
break
# Print result and log
if result:
_print(F"{Fore.GREEN}Backup {container.restore_tar_file} for {container.name} successfully restored.{Style.RESET_ALL}")
if not utils.no_log and config_list['log']['logging']:
log.log(F"Restore backup ; {container.name} ; {container.restore_tar_file_path} ; SUCCESS")
else:
_print(F"{Fore.RED}Could not restore {container.restore_tar_file} for {container.name}.{Style.RESET_ALL}")
for func, traceback in container.exceptions.items():
_print()
_print(F"{Fore.YELLOW}Exception occurred in method: Container.{func}(){Style.RESET_ALL}")
_print(traceback)
_print()
if not utils.no_log and config_list['log']['logging']:
log.log(F"Restore backup ; {container.name} ; {container.restore_tar_file_path} ; FAIL")
def upgrade():
# Set flags
utils.set_flags(sys.argv)
# Load config
config = Path(__file__).parent / "config.yml"
with open(config) as file:
# Load config
config_list = yaml.full_load(file)
log = Log(config_list['log']['log_dir'])
containers = Container.instantiate_containers(config_list)
# If any container names were passed as parameters, do only upgrade them
containers_wanted = {
name: container
for name, container in containers.items() if name in sys.argv
}
if containers_wanted:
containers = containers_wanted
# If no container was chosen ask for it
elif not utils.all_containers:
containers_to_choose_from = [
container.name for container in containers.values()
]
terminal_menu = TerminalMenu(
containers_to_choose_from,
title="Which Nextcloud instance do you want to "
"upgrade?")
choice_index = terminal_menu.show()
containers = {
containers_to_choose_from[choice_index]:
containers.get(containers_to_choose_from[choice_index])
}
# Loop through Nextcloud container instances
container: Container
for container in containers.values():
go_on = True
# Make a backup
if not utils.no_backup:
utils.keep_maintenance_mode = True
go_on = backup.backup()
if go_on:
# Make the upgrade
utils.keep_maintenance_mode = True if "--maintenance" in sys.argv else False
_print("----------------------------------------------")
_print(
F"Start upgrade for {container.name} at {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
)
result = container.upgrade()
if result == 1:
_print(
F"{Fore.GREEN}{container.name} upgraded successfully{Style.RESET_ALL}"
)
upgrade_status = True
elif result == 2:
_print(
F"{Fore.GREEN}No upgrades available for {container.name}.{Style.RESET_ALL}"
)
upgrade_status = True
else:
_print(
F"{Fore.RED}Upgrade for {container.name} failed{Style.RESET_ALL}"
)
for func, traceback in container.exceptions.items():
_print()
_print(
F"{Fore.YELLOW}Exception occurred in method: Container.{func}(){Style.RESET_ALL}"
)
_print(traceback)
_print()
# Log upgrade
if not utils.no_log and config_list['log']['logging']:
if upgrade_status:
log.log(F"Upgrade ; {container.name} ; SUCCESS")
else:
log.log(F"Upgrade ; {container.name} ; FAIL")
if len(log.exceptions) > 0:
for func, traceback in log.exceptions.items():
_print()
_print(
F"{Fore.YELLOW}Exception occurred in method: Log.{func}(){Style.RESET_ALL}"
)
_print(traceback)
_print()
def install_docker(request, domain, password):
container = Container.create_new_container(domain, password)
return HttpResponse(u"Youpi, ça a marché ! %s" % container.ip)
def list():
containers = [
Container(container) for container in cli.containers(all=True)
]
return containers_list('containers/list.html', containers)
from database import Base, engine, Session from models import Node, Container, ContainerNodeAssoc Base.metadata.create_all(engine) session = Session() n1 = Container(parent_id=None, data="container1") n2 = Container(parent_id=None, data="container2") c1 = Container(data="child1") c2 = Container(data="child2") c3 = Container(data="child3") c4 = Container(data="child4") c5 = Container(data="child5") c6 = Container(data="child6") node1 = Node(name="node1") node2 = Node(name="node2") node3 = Node(name="node3") node4 = Node(name="node4") node5 = Node(name="node5") node6 = Node(name="node6") # populate the assoc table # node1 in container1 node1_container1 = ContainerNodeAssoc(container=c1, node=node1) node2_container2 = ContainerNodeAssoc(container=c2, node=node2) node3_container3 = ContainerNodeAssoc(container=c3, node=node3) node4_container4 = ContainerNodeAssoc(container=c4, node=node4) node5_container5 = ContainerNodeAssoc(container=c5, node=node5)
def obj_create(self, bundle, request=None, **kwargs):
'''
Function to create a bundle via the API.
Arguments (in JSON) are as follows:
rec_id - the name for the Bundle (REQUIRED)
content - the content in JSON format (or URL REQUIRDE)
URL - a URL reference to the JSON content of the bundle * (OR content REQUIRED)
public - indicator whether the Bundle to be made public, False if not provided
licenses - a list of names corresponding to License objects in the DB
submission - a file submitted which represents the Bundle **
* - The URL is only opened and it's content parsed if and only if 'content' is empty
** - The file content is not parsed at all, it is just saved for later usage and
if file is submitted the HTTP call should be in 'multipart' encoding format.
'''
try:
'''Try to read the content else download and parse the URL file '''
prov_bundle = ProvBundle()
if bundle.data['content']:
prov_bundle._decode_JSON_container(bundle.data['content'])
else:
source = urlopen(bundle.data['url'], timeout=5)
content = source.read()
source.close()
prov_bundle._decode_JSON_container(loads(content))
container = Container.create(bundle.data['rec_id'], prov_bundle, request.user)
except:
raise ImmediateHttpResponse(HttpBadRequest())
save = False
if 'public' in bundle.data:
container.public = bundle.data['public']
save = True
if bundle.data['public']:
assign('view_container', Group.objects.get(id=PUBLIC_GROUP_ID), container)
if 'licenses' in bundle.data:
for title in bundle.data['licenses']:
try:
lic = License.objects.get(title=title)
container.license.add(lic)
save = True
except License.DoesNotExist:
pass
if 'submission' in request.FILES:
file_sub = request.FILES['submission']
sub = Submission.objects.create()
sub.content.save(sub.timestamp.strftime('%Y-%m-%d%H-%M-%S')+file_sub._name, file_sub)
container.submission = sub
save = True
if 'url' in bundle.data:
container.url = bundle.data['url']
save = True
if save:
container.save()
bundle.obj = container
return bundle