def run(self): try: ips = [] nets = [] for l in self.instance.links: ips.append(l['ip']) net = NetworkNode.fetchone(gid=l['network']['gid'], slice_id=self.lab_slice.id) nets.append(net) instance = self.instance public_ip, attrs = self.openstack.create_instance( instance.name, nets, ips, instance.configurations, self.sec_group_id, instance.image, instance.flavor) instance.update(status='active', public_ip=public_ip, cloud_attrs=attrs) except Exception as ex: error_type = 'Create instances error' error_msgs = [error_type + ': ' + str(ex)] lab = Lab.fetchone(id=self.lab_id) lab.update(status='deployfailed', error_msgs=lab.error_msgs + error_msgs) raise Exception(error_type + str(error_msgs))
def delete_lab(lab_id, last_jobs_ids): if len(last_jobs_ids) > 0: timeout = 30 while True: all_completed = True for id in last_jobs_ids: job = queue.fetch_job(id) print(job.get_status()) if job.get_status() != JobStatus.FINISHED: all_completed = False if all_completed: break time.sleep(5) timeout = timeout - 5 if timeout == 0: break lab = Lab.fetchone(id=lab_id) if timeout == 0: error_type = 'Time out error when trying to delete lab' error_msgs = [error_type] lab.update(status='destroyfailed', error_msgs=lab.error_msgs + error_msgs) else: lab.delete()
def create_networks(cloudconfig, lab_id, lab_slice, topo): try: openstack = Openstack(cloudconfig.detail['openstackAuthURL'], cloudconfig.detail['openstackProject'], cloudconfig.detail['openstackUser'], cloudconfig.detail['openstackPassword']) threads = [] for n in topo['networks']: new_net = NetworkNode.insert(name=n['name'], cidr=n['cidr'], status='deploying', x=n['x'], y=n['y'], slice_id=lab_slice.id, gid=n['gid']) t = CreateNetThread(openstack, new_net) t.start() threads.append(t) for t in threads: t.join() except Exception as ex: error_type = 'Create networks error' error_msgs = [error_type + ': ' + str(ex)] lab = Lab.fetchone(id=lab_id) lab.update(status='deployfailed', error_msgs=lab.error_msgs + error_msgs) raise Exception( error_type ) # Raise exception to not execute the next job in the dependency link
def set_lab_active(lab_id, last_jobs_ids): if len(last_jobs_ids) > 0: timeout = 20 while True: all_completed = True for id in last_jobs_ids: job = queue.fetch_job(id) if job.get_status() != JobStatus.FINISHED: all_completed = False if all_completed: break time.sleep(5) timeout = timeout - 5 if timeout == 0: break lab = Lab.fetchone(id=lab_id) if timeout == 0: error_type = 'Time out error when trying to set lab active' error_msgs = [error_type] lab.update(status='deployfailed', error_msgs=lab.error_msgs + error_msgs) else: lab.update(status='active', error_msgs=[])
def get(self, id): lab = Lab.fetchone(id=id) if lab is None: return jsonify(message="Lab doesn't existed"), 410 if lab.status == 'inactive': return jsonify({ 'id': lab.id, 'name': lab.name, 'status': lab.status }) else: slices = [] for sl in Slice.fetchall(lab_id=lab.id): user = User.fetchone(id=sl.user_id) slices.append({ 'id': sl.id, 'name': sl.name, 'status': sl.status, 'username': user.fullname }) return jsonify({ 'id': lab.id, 'name': lab.name, 'status': lab.status, 'slices': slices, 'errors': lab.error_msgs })
def update_allowed_address_pairs(cloudconfig, lab_id, lab_slice, topo): try: openstack = Openstack(cloudconfig.detail['openstackAuthURL'], cloudconfig.detail['openstackProject'], cloudconfig.detail['openstackUser'], cloudconfig.detail['openstackPassword']) mac_regex = '^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$' ip_cidr_regex = '^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4]' \ '[0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)|(([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-' \ 'f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-' \ '5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5' \ ']|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}((' \ '(:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0' \ '-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]' \ '{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0' \ '-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d' \ '|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1' \ ',4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d' \ '|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1' \ '\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(\/[0-9]{1,2})*$' for link in topo['links']: address_pairs = [] network = NetworkNode.fetchone(gid=link['network']['gid'], slice_id=lab_slice.id) if link['target']['type'].lower() == 'instance': device = Instance.fetchone(gid=link['target']['gid'], slice_id=lab_slice.id) elif link['target']['type'].lower() == 'router': device = Router.fetchone(gid=link['target']['gid'], slice_id=lab_slice.id) else: continue for raw_address_pair in link.get('allowedAddressPairs', []): mac_address, ip_address, _ = (raw_address_pair + ',,').split( ',', 2) if ip_address.strip() == '': ip_address = mac_address mac_address = '' if (mac_address == '' or re.match( mac_regex, mac_address.strip())) and re.match( ip_cidr_regex, ip_address.strip()): address_pair = {'ip_address': ip_address} if mac_address != '': address_pair['mac_address'] = mac_address address_pairs.append(address_pair) if address_pairs: openstack.update_allowed_address_pairs( network, device.cloud_attrs['id'], address_pairs) except Exception as ex: error_type = 'Update allowed address pairs error' error_msgs = [error_type + ': ' + str(ex)] lab = Lab.fetchone(id=lab_id) lab.update(status='deployfailed', error_msgs=lab.error_msgs + error_msgs) raise Exception( error_type ) # Raise exception to not execute the next job in the dependency link
def destroy(self, id): lab = Lab.fetchone(id=id) lab.update(status='destroying') cloudconfig = CloudConfig.fetchone(lab_id=id) if cloudconfig is None: """No cloud config has been set. Just delete the lab instance""" lab.delete() return jsonify(message='ok') if cloudconfig.provider == 'Openstack': return _destroy_openstack(id, cloudconfig)
def set_slice_active(lab_id, lab_slice): try: lab_slice.update(status='active') except Exception as ex: error_type = 'Set slice active error' error_msgs = [error_type + ': ' + str(ex)] lab = Lab.fetchone(id=lab_id) lab.update(status='deployfailed', error_msgs=lab.error_msgs + error_msgs) raise Exception( error_type ) # Raise exception to not execute the next job in the dependency link
def run(self): try: if self.router.status == 'active': self.router.update(status='destroying') self.openstack.delete_instance(self.router.cloud_attrs['id']) self.router.delete() except Exception as ex: error_type = 'Delete routers error' error_msgs = [error_type + ': ' + str(ex)] lab = Lab.fetchone(id=self.lab_id) lab.update(status='destroyfailed', error_msgs=lab.error_msgs + error_msgs)
def create_routers(cloudconfig, lab_id, lab_slice, topo, create_sec_group_job_id): try: openstack = Openstack(cloudconfig.detail['openstackAuthURL'], cloudconfig.detail['openstackProject'], cloudconfig.detail['openstackUser'], cloudconfig.detail['openstackPassword']) sec_group_id = queue.fetch_job(create_sec_group_job_id).result routers = topo['routers'] for s in routers: links = _extract_links(s, topo) configurations, password = _extract_configurations( lab_id, lab_slice, s, topo) Router.insert(name=s['name'], status='deploying', x=s['x'], y=s['y'], gid=s['gid'], slice_id=lab_slice.id, image=s['image'], flavor=s['flavor'], links=links, configurations=configurations, password=password) # Actually deployment threads = [] for router in Router.fetchall(slice_id=lab_slice.id): t = CreateRouterThread(openstack, lab_id, router, lab_slice, sec_group_id) t.start() threads.append(t) for t in threads: t.join() except Exception as ex: error_type = 'Create routers error' error_msgs = [error_type + ': ' + str(ex)] lab = Lab.fetchone(id=lab_id) lab.update(status='deployfailed', error_msgs=lab.error_msgs + error_msgs) raise Exception( error_type ) # Raise exception to not execute the next job in the dependency link
def delete_sec_group(cloudconfig: CloudConfig, lab_id, lab_slice: Slice): try: openstack = Openstack(cloudconfig.detail['openstackAuthURL'], cloudconfig.detail['openstackProject'], cloudconfig.detail['openstackUser'], cloudconfig.detail['openstackPassword']) if lab_slice.cloud_attrs.get('sec_group_id') is not None: openstack.delete_security_group(lab_slice.name) except Exception as ex: error_type = 'Delete security group error' error_msgs = [error_type + ': ' + str(ex)] lab = Lab.fetchone(id=lab_id) lab.update(status='destroyfailed', error_msgs=lab.error_msgs + error_msgs) raise Exception( error_type ) # Raise exception to not execute the next job in the dependency link
def index(self): """ List ONLY slices that a user has been added to""" with Select(Slice, 'user_id = %s') as select: select.execute((g.user['id'], )) slices = select.fetchall() ret = [] for sl in slices: lab = Lab.fetchone(id=sl.lab_id) ret.append({ 'id': sl.id, 'status': sl.status, 'lab': { 'name': lab.name, 'description': lab.description, } }) return jsonify(sorted(ret, key=lambda i: i['id'], reverse=True))
def deploy(self, id): cloudconfig_id = request.get_json()['cloudConfigId'] cloudconfig = CloudConfig.fetchone(id=cloudconfig_id) users = request.get_json()['users'] lab = Lab.fetchone(id=id) lab.update(status='deploying') """Chances are you are redeploying a lab, whose slices are already created""" slices = Slice.fetchall(lab_id=lab.id) """Otherwise, create slices for the lab""" if len(slices) == 0: for index, user in enumerate(users): new_slice = Slice.insert(lab_id=lab.id, status='deploying', user_id=user['id'], name=lab.name + ' / slice_' + str(index), cloud_attrs={}) slices.append(new_slice) scenario = Scenario.fetchone(id=lab.scenario_id) if cloudconfig.provider == 'Openstack': return _deploy_openstack(id, cloudconfig, slices, scenario)
def _extract_configurations(lab_id, slice: Slice, instance: Dict[str, Any], topo: Dict[str, Any]) \ -> Tuple[List[Dict[str, Any]], str]: """extract configurations of an instance base on the topology the extracted configurations is a dict of configuration name ('name') and and rendering parameters ('params') rendering parameters can be None""" lab = Lab.fetchone(id=lab_id) configurations = [] # configurations.append({"name": "staticroute", "params": _extract_static_route(instance, topo)}) configurations.append({"name": "add-local-host", "params": {}}) if instance.get('type') == "Router": """ Get the number of interfaces """ interfaces_count = sum(1 for link in topo['links'] if link['target']['gid'] == instance['gid']) configurations.append({ "name": "shorewall", "params": { "interfaces_count": interfaces_count } }) password = None for conf in instance.get('configurations', []): params: Dict[str, Any] = {} if conf == "Enable password authentication" or conf == "noVNC": if password is None: password = randomword(8) params['password'] = password elif conf == "grr-client": user = User.fetchone(id=slice.user_id) params['labels'] = [lab.name, user.email, instance['name']] configurations.append({"name": conf, "params": params}) return configurations, password
def create_sec_group(cloudconfig: CloudConfig, lab_id, lab_slice: Slice, scenario): try: openstack = Openstack(cloudconfig.detail['openstackAuthURL'], cloudconfig.detail['openstackProject'], cloudconfig.detail['openstackUser'], cloudconfig.detail['openstackPassword']) sec_group_id = openstack.create_security_group(lab_slice.name) new_cloud_attrs = lab_slice.cloud_attrs new_cloud_attrs['sec_group_id'] = sec_group_id lab_slice.update(cloud_attrs=new_cloud_attrs.value) for rule in scenario.sg_rules: _add_security_group_rule(openstack, sec_group_id, rule) return sec_group_id except Exception as ex: error_type = 'Create security group error' error_msgs = [error_type + ': ' + str(ex)] lab = Lab.fetchone(id=lab_id) lab.update(status='deployfailed', error_msgs=lab.error_msgs + error_msgs) raise Exception( error_type ) # Raise exception to not execute the next job in the dependency link
def delete_routers(cloudconfig, lab_id, lab_slice): try: openstack = Openstack(cloudconfig.detail['openstackAuthURL'], cloudconfig.detail['openstackProject'], cloudconfig.detail['openstackUser'], cloudconfig.detail['openstackPassword']) routers = Router.fetchall(slice_id=lab_slice.id) threads = [] for router in routers: t = DeleteRouterThread(openstack, lab_id, router) t.start() threads.append(t) for t in threads: t.join() except Exception as ex: error_type = 'Delete routers error' error_msgs = [error_type + ': ' + str(ex)] lab = Lab.fetchone(id=lab_id) lab.update(status='destroyfailed', error_msgs=lab.error_msgs + error_msgs) raise Exception( error_type ) # Raise exception to not execute the next job in the dependency link
def test_deploy_destroy_lab(post_lab, root_user_fixture): user_id, email, token = root_user_fixture lab_id = post_lab # Create a cloud config rv = test_client.post('/api/cloudconfigs/', headers=api_headers(email, token), data=json.dumps({ 'cloudDetail': CLOUD_DETAIL, 'provider': 'Openstack', 'labId': lab_id })) assert rv.status_code == 200 cloudconfig_id = json.loads(rv.data)['id'] users = [{'id': user_id}] deploy_url = '/api/labs/{0}/deploy'.format(lab_id, ) rv = test_client.post(deploy_url, headers=api_headers(email, token), data=json.dumps({ 'cloudConfigId': cloudconfig_id, 'users': users })) assert rv.status_code == 200 # assert lab is active after a while timeout = 200 while True: lab = Lab.fetchone(id=lab_id) if lab.status == 'active': break print(lab.status) print(lab.error_msgs) assert lab.status != 'deployfailed' time.sleep(20) timeout = timeout - 20 if timeout <= 0: break assert timeout > 0, "Deployment timeout" # destroy lab destroy_url = '/api/labs/{0}/destroy'.format(lab_id, ) rv = test_client.post(destroy_url, headers={ 'Content-Type': 'application/json', 'Email': email, 'Authorization': token }) timeout = 200 while True: lab = Lab.fetchone(id=lab_id) if lab is None: break print(lab.status) print(lab.error_msgs) assert lab.status != 'destroyfailed' time.sleep(20) timeout = timeout - 20 if timeout <= 0: break assert timeout > 0, "Deletion timeout"
def get(self, id): sl = Slice.fetchone(id=id) # only user or the lab creator can access if sl.user_id != g.user['id']: lab = Lab.fetchone(id=sl.lab_id) if lab.owner_id != g.user['id']: return jsonify( message="Access denied to this slice resource"), 403 # get networks with Select(NetworkNode, 'slice_id=%s and status!=%s') as select: select.execute((sl.id, 'inactive')) networks = [{ 'id': n.id, 'name': n.name, 'cidr': n.cidr, 'status': n.status, 'x': n.x, 'y': n.y, 'type': 'NetworkNode' } for n in select.fetchall()] instances = [] links = [] # get instances with Select( Instance, 'instances.slice_id=%s and instances.status!=%s') as select: select.execute((sl.id, 'inactive')) results = select.fetchall() for res in results: instance = res instances.append({ 'id': instance.id, 'name': instance.name, 'public_ip': instance.public_ip, 'status': instance.status, 'password': instance.password, 'x': instance.x, 'y': instance.y, 'configurations': instance.configurations. value, # TODO: remove .value here after implement it in postgrespy 'type': 'Instance' }) for link in instance.links: links.append({ 'gid': link['gid'], 'ip': link['ip'], 'network': link['network'], 'target': link['target'], 'type': 'NetworkLink' }) # get routers routers = [] with Select(Router, 'routers.slice_id=%s and routers.status!=%s') as select: select.execute((sl.id, 'inactive')) results = select.fetchall() for res in results: router = res routers.append({ 'id': router.id, 'name': router.name, 'public_ip': router.public_ip, 'status': router.status, 'password': router.password, 'x': router.x, 'y': router.y, 'configurations': router.configurations. value, # TODO: remove .value here after implement it in postgrespy 'type': 'Router' }) for link in router.links: links.append({ 'gid': link['gid'], 'ip': link['ip'], 'network': link['network'], 'target': link['target'], 'type': 'NetworkLink' }) user = User.fetchone(id=sl.user_id) return jsonify({ 'status': sl.status, 'name': sl.name, 'username': user.fullname, 'networks': networks, 'instances': instances, 'routers': routers, 'links': links })