def test_machine_exists_state_exists(self): mac = "00:00:00:00:00:00" state = MachineStates.booting with session_commit(sess_maker=self.sess_maker) as session: uuid = "b7f5f93a-b029-475f-b3a4-479ba198cb8a" machine = Machine(uuid=uuid) session.add(machine) machine_id = session.query(Machine).filter(Machine.uuid == uuid).first().id session.add( MachineInterface(machine_id=machine_id, mac=mac, netmask=1, ipv4="10.10.10.10", cidrv4="127.0.0.1/8", as_boot=True, gateway="1.1.1.1", name="lol")) session.commit() msr = MachineStateRepository(sess_maker=self.sess_maker) msr.update(mac, state) new_state = MachineStates.discovery msr.update(mac, new_state) with session_commit(sess_maker=self.sess_maker) as session: res = session.query(MachineCurrentState).filter(MachineCurrentState.machine_mac == mac).first() self.assertEqual(mac, res.machine_mac) self.assertEqual(new_state, res.state_name) self.assertEqual(machine_id, res.machine_id) updated_date = res.updated_date ret = msr.fetch(10) self.assertEqual([{ "fqdn": None, "mac": mac, "state": new_state, "date": updated_date }], ret)
def test_34(self): with session_commit(sess_maker=self.sess_maker) as session: rq = "uuid=%s&mac=%s&os=installed" % ( posts.M03["boot-info"]["uuid"], posts.M03["boot-info"]["mac"]) i = crud.InjectLifecycle(session, request_raw_query=rq) i.refresh_lifecycle_ignition(True) with session_commit(sess_maker=self.sess_maker) as session: j = crud.InjectLifecycle(session, request_raw_query=rq) j.refresh_lifecycle_ignition(False) f = crud.FetchLifecycle(sess_maker=self.sess_maker) self.assertFalse( f.get_ignition_uptodate_status(posts.M03["boot-info"]["mac"])) self.assertEqual(3, len(f.get_all_updated_status()))
def test_36(self): with session_commit(sess_maker=self.sess_maker) as session: rq = "uuid=%s&mac=%s&os=installed" % ( posts.M03["boot-info"]["uuid"], posts.M03["boot-info"]["mac"]) i = crud.InjectLifecycle(session, request_raw_query=rq) i.apply_lifecycle_rolling(True) f = crud.FetchLifecycle(sess_maker=self.sess_maker) status = f.get_rolling_status(posts.M03["boot-info"]["mac"]) self.assertTrue(status[0]) self.assertEqual("kexec", status[1]) with session_commit(sess_maker=self.sess_maker) as session: n = crud.InjectLifecycle(session, rq) n.apply_lifecycle_rolling(False) f = crud.FetchLifecycle(sess_maker=self.sess_maker) r = f.get_rolling_status(posts.M03["boot-info"]["mac"]) self.assertFalse(r[0]) self.assertEqual("kexec", r[1]) with session_commit(sess_maker=self.sess_maker) as session: n = crud.InjectLifecycle(session, rq) n.apply_lifecycle_rolling(True, "reboot") f = crud.FetchLifecycle(sess_maker=self.sess_maker) r = f.get_rolling_status(posts.M03["boot-info"]["mac"]) self.assertTrue(r[0]) self.assertEqual("reboot", r[1]) with session_commit(sess_maker=self.sess_maker) as session: n = crud.InjectLifecycle(session, rq) n.apply_lifecycle_rolling(True, "poweroff") f = crud.FetchLifecycle(sess_maker=self.sess_maker) r = f.get_rolling_status(posts.M03["boot-info"]["mac"]) self.assertTrue(r[0]) self.assertEqual("poweroff", r[1]) with session_commit(sess_maker=self.sess_maker) as session: n = crud.InjectLifecycle(session, rq) with self.assertRaises(LookupError): n.apply_lifecycle_rolling(True, "notpossible") f = crud.FetchLifecycle(sess_maker=self.sess_maker) r = f.get_rolling_status(posts.M03["boot-info"]["mac"]) self.assertTrue(r[0]) self.assertEqual("poweroff", r[1])
def report_lifecycle_coreos_install(status, request_raw_query): """ Lifecycle CoreOS Install Report the status of a CoreOS install by MAC --- tags: - lifecycle responses: 200: description: CoreOS Install report schema: type: dict """ app.logger.info("%s %s" % (request.method, request.url)) if status.lower() == "success": success = True elif status.lower() == "fail": success = False else: app.logger.error("%s %s" % (request.method, request.url)) return "success or fail != %s" % status.lower(), 403 with session_commit(sess_maker=sess_maker) as session: inject = crud.InjectLifecycle(session, request_raw_query=request_raw_query) inject.refresh_lifecycle_coreos_install(success) registry.machine_state.update( mac=tools.get_mac_from_raw_query(request_raw_query), state=MachineStates.installation_succeed if success else MachineStates.installation_failed) return jsonify({ "success": success, "request_raw_query": request_raw_query }), 200
def get_available_machines(self): available_machines = [] with session_commit(sess_maker=self.__sess_maker) as session: for m in session.query(Machine) \ .join(MachineInterface) \ .options(joinedload("schedules")) \ .options(joinedload("interfaces")) \ .options(joinedload("disks")) \ .filter(MachineInterface.as_boot == True): # TODO find a way to support cockroach and SQLite without this if if not m.schedules: available_machines.append({ "mac": m.interfaces[0].mac, "ipv4": m.interfaces[0].ipv4, "cidrv4": m.interfaces[0].cidrv4, "as_boot": m.interfaces[0].as_boot, "name": m.interfaces[0].name, "fqdn": m.interfaces[0].fqdn, "netmask": m.interfaces[0].netmask, "created_date": m.created_date, "disks": [{ "path": k.path, "size-bytes": k.size } for k in m.disks], }) return available_machines
def lifecycle_rolling_delete(request_raw_query): """ Lifecycle Rolling Update Disable the current policy for a given machine by UUID or MAC --- tags: - lifecycle parameters: - name: request_raw_query in: path description: Pass the mac as 'mac=<mac>' required: true type: string responses: 200: description: Rolling Update is not enable schema: type: dict """ app.logger.info("%s %s" % (request.method, request.url)) with session_commit(sess_maker=sess_maker) as session: life = crud.InjectLifecycle(session, request_raw_query) life.apply_lifecycle_rolling(False, None) return jsonify({ "enable": False, "request_raw_query": request_raw_query }), 200
def get_all_updated_status(self): status = [] with session_commit(sess_maker=self.sess_maker) as session: for machine in session.query(Machine)\ .join(LifecycleIgnition)\ .join(MachineInterface)\ .filter(MachineInterface.as_boot == True): status.append({ "up-to-date": machine.lifecycle_ignition[0].up_to_date, "fqdn": machine.interfaces[0].fqdn, "mac": machine.interfaces[0].mac, "cidrv4": machine.interfaces[0].cidrv4, "created_date": machine.created_date, "updated_date": machine.updated_date, "last_change_date": machine.lifecycle_ignition[0].last_change_date, }) return status
def test_one_machine_with_only_interfaces(self): mac = "00:00:00:00:00:00" with session_commit(sess_maker=self.sess_maker) as session: uuid = "b7f5f93a-b029-475f-b3a4-479ba198cb8a" machine = Machine(uuid=uuid) session.add(machine) machine_id = session.query(Machine).filter( Machine.uuid == uuid).first().id session.add( MachineInterface(machine_id=machine_id, mac=mac, netmask=1, ipv4="10.10.10.10", cidrv4="127.0.0.1/8", as_boot=True, gateway="1.1.1.1", name="lol")) session.commit() expect = list() expect.append({ 'CIDR': '127.0.0.1/8', 'LastReport': None, 'UpdateStrategy': 'Disable', 'LastChange': None, 'MAC': '00:00:00:00:00:00', 'UpToDate': None, 'FQDN': None, 'DiskProfile': 'inMemory', 'LastState': None, 'Roles': '' }) ui = user_interface.UserInterfaceRepository(sess_maker=self.sess_maker) self.assertCountEqual(expect, ui.get_machines_overview())
def get_all_rolling_status(self): life_roll_list = [] with session_commit(sess_maker=self.sess_maker) as session: for machine in session.query(Machine) \ .join(LifecycleRolling) \ .join(MachineInterface) \ .options(joinedload("interfaces")) \ .options(joinedload("lifecycle_rolling")) \ .filter(MachineInterface.as_boot == True): try: life_roll_list.append({ "mac": machine.interfaces[0].mac, "fqdn": machine.interfaces[0].fqdn, "cidrv4": machine.interfaces[0].cidrv4, "enable": bool(machine.lifecycle_rolling[0].enable), "created_date": machine.lifecycle_rolling[0].created_date, "updated_date": machine.lifecycle_rolling[0].updated_date }) except IndexError: pass return life_roll_list
def update(self, mac: str, state: str): with session_commit(sess_maker=self.__sess_maker) as session: state_machine = session.query(MachineCurrentState) \ .filter(MachineCurrentState.machine_mac == mac) \ .one_or_none() machine_interface = session.query(MachineInterface) \ .filter(MachineInterface.mac == mac) \ .one_or_none() now = datetime.datetime.utcnow() machine_id = None if machine_interface is not None: machine_id = machine_interface.machine_id if not state_machine: logger.debug( "machine with mac: %s doesn't exist in table %s: creating with state %s" % ( mac, MachineCurrentState.__tablename__, state)) self._update_state(session, MachineCurrentState( machine_id=machine_id, state_name=state, machine_mac=mac, created_date=now, updated_date=now, )) else: state_machine.state_name = state state_machine.machine_id = machine_id state_machine.updated_date = now self._update_state(session, state_machine)
def get_playbook(self): """ Get and reproduce the data sent inside the db from an API level :return: """ playbook = [] with session_commit(sess_maker=self.sess_maker) as session: for schedule_type in [ ScheduleRoles.kubernetes_control_plane, ScheduleRoles.kubernetes_node ]: for machine in session.query(Machine).filter( Machine.schedules.any(Schedule.role == schedule_type)): discovery_data = self._construct_discovery(machine) schedule_data = self._construct_schedule( discovery_data["boot-info"]["mac"], schedule_type) playbook.append({ "data": discovery_data, "route": "/discovery" }) playbook.append({ "data": schedule_data, "route": "/scheduler" }) return playbook
def upsert(self, discovery_data: dict): discovery_data = self._lint_discovery_data(discovery_data) now = datetime.datetime.utcnow() new = True with session_commit(sess_maker=self.__sess_maker) as session: machine = session.query(Machine) \ .filter(Machine.uuid == discovery_data["boot-info"]["uuid"]) \ .first() if machine: new = False machine.updated_date = now self._delete_all_attached(session, machine) else: machine = Machine(uuid=discovery_data["boot-info"]["uuid"], created_date=now, updated_date=now) session.add(machine) session.flush() for d in discovery_data["disks"]: session.add( MachineDisk(path=d["path"], size=d["size-bytes"], machine_id=machine.id)) self._insert_network(session, machine, discovery_data) session.commit() return new
def setUpClass(cls): db_uri = 'postgresql+psycopg2://localhost/enjoliver_testing' cls.engine = create_engine(db_uri) cls.sess_maker = sessionmaker(bind=cls.engine) cls.repositories = RepositoryRegistry(sess_maker=cls.sess_maker) cls.init_db() with session_commit(sess_maker=cls.sess_maker) as session: ops.health_check(session, time.time(), "unittest")
def get_machines_by_role(self, role: str): machines = [] with session_commit(sess_maker=self.__sess_maker) as session: for machine in session.query(Machine) \ .join(Schedule) \ .filter(Schedule.role == role): machines.append(self._construct_machine_dict(machine, role)) return machines
def get_all_schedules(self): result = dict() with session_commit(sess_maker=self.__sess_maker) as session: for machine in session.query(Machine): if machine.schedules: result[machine.boot_interface.mac] = [ k.role for k in machine.schedules ] return result
def test_no_machine(self): mdr = MachineDiscoveryRepository(self.sess_maker) mdr.upsert(posts.M01) with session_commit(sess_maker=self.sess_maker) as session: self.assertEqual(1, session.query(Machine).count()) self.assertEqual(1, session.query(MachineInterface).count()) self.assertEqual(1, session.query(MachineDisk).count()) self.assertEqual(1, session.query(Chassis).count()) self.assertEqual(1, session.query(ChassisPort).count())
def test_35(self): with session_commit(sess_maker=self.sess_maker) as session: rq = "uuid=%s&mac=%s&os=installed" % ( posts.M03["boot-info"]["uuid"], posts.M03["boot-info"]["mac"]) i = crud.InjectLifecycle(session, request_raw_query=rq) i.refresh_lifecycle_coreos_install(True) f = crud.FetchLifecycle(sess_maker=self.sess_maker) self.assertTrue( f.get_coreos_install_status(posts.M03["boot-info"]["mac"])) self.assertEqual(1, len(f.get_all_coreos_install_status()))
def get_role_ip_list(self, role: str): ips = [] with session_commit(sess_maker=self.__sess_maker) as session: for machine in session.query(Machine) \ .options(joinedload("interfaces")) \ .join(MachineInterface) \ .join(Schedule) \ .filter(Schedule.role == role, MachineInterface.as_boot == True): ips.append(machine.interfaces[0].ipv4) return ips
def get_ignition_uptodate_status(self, mac: str): with session_commit(sess_maker=self.sess_maker) as session: lf = session.query(LifecycleIgnition)\ .join(Machine)\ .join(MachineInterface)\ .filter(MachineInterface.mac == mac)\ .first() if lf: return lf.up_to_date else: return None
def get_coreos_install_status(self, mac: str): with session_commit(sess_maker=self.sess_maker) as session: lci = session.query(LifecycleCoreosInstall)\ .join(Machine)\ .join(MachineInterface)\ .filter(MachineInterface.mac == mac)\ .first() if lci: return lci.success else: return None
def test_no_machine_no_state(self): mac = "00:00:00:00:00:00" state = MachineStates.booting msr = MachineStateRepository(sess_maker=self.sess_maker) msr.update(mac, state) with session_commit(sess_maker=self.sess_maker) as session: res = session.query(MachineCurrentState).filter(MachineCurrentState.machine_mac == mac).first() self.assertEqual(mac, res.machine_mac) self.assertEqual(state, res.state_name) self.assertEqual(None, res.machine_id)
def test_no_machine_remove_disks(self): mdr = MachineDiscoveryRepository(self.sess_maker) mdr.upsert(posts.M01) with session_commit(sess_maker=self.sess_maker) as session: self.assertEqual(1, session.query(Machine).count()) self.assertEqual(1, session.query(MachineInterface).count()) self.assertEqual(1, session.query(MachineDisk).count()) self.assertEqual(1, session.query(Chassis).count()) self.assertEqual(1, session.query(ChassisPort).count()) without_disks = copy.deepcopy(posts.M01) without_disks["disks"] = None mdr.upsert(without_disks) with session_commit(sess_maker=self.sess_maker) as session: self.assertEqual(1, session.query(Machine).count()) self.assertEqual(1, session.query(MachineInterface).count()) self.assertEqual(0, session.query(MachineDisk).count()) self.assertEqual(1, session.query(Chassis).count()) self.assertEqual(1, session.query(ChassisPort).count())
def change_lifecycle_rolling(request_raw_query): """ Lifecycle Rolling Update Change the current policy for a given machine by MAC --- tags: - lifecycle parameters: - name: request_raw_query in: path description: Pass the mac as 'mac=<mac>' required: true type: string responses: 200: description: Rolling Update is enable schema: type: dict 401: description: Mac address is not in database schema: type: dict """ app.logger.info("%s %s" % (request.method, request.url)) try: strategy = json.loads(request.get_data())["strategy"] app.logger.info("%s %s rolling strategy: setting to %s" % (request.method, request.url, strategy)) except (KeyError, ValueError): # JSONDecodeError is a subclass of ValueError # Cannot use JSONDecodeError because the import is not consistent between python3.X app.logger.info( "%s %s rolling strategy: setting default to kexec" % (request.method, request.url)) strategy = "kexec" with session_commit(sess_maker=sess_maker) as session: try: life = crud.InjectLifecycle(session, request_raw_query) life.apply_lifecycle_rolling(True, strategy) return jsonify({ "enable": True, "request_raw_query": request_raw_query, "strategy": strategy }), 200 except AttributeError: return jsonify({ "enable": None, "request_raw_query": request_raw_query, "strategy": strategy }), 401
def test_one_machine_full_scheduled_with_strategy_disable(self): mac = "00:00:00:00:00:00" with session_commit(sess_maker=self.sess_maker) as session: uuid = "b7f5f93a-b029-475f-b3a4-479ba198cb8a" machine = Machine(uuid=uuid) session.add(machine) machine_id = session.query(Machine).filter( Machine.uuid == uuid).first().id session.add( MachineInterface(machine_id=machine_id, mac=mac, netmask=1, ipv4="10.10.10.10", cidrv4="127.0.0.1/8", as_boot=True, gateway="1.1.1.1", name="lol")) session.add( MachineDisk(path="/dev/sda", size=1024 * 1024 * 1024, machine_id=machine_id)) session.add( MachineCurrentState(machine_id=machine_id, machine_mac=mac, state_name=MachineStates.discovery)) session.add( Schedule(machine_id=machine_id, role=ScheduleRoles.kubernetes_control_plane)) session.add( LifecycleRolling(machine_id=machine_id, strategy="kexec", enable=False)) session.commit() expect = list() expect.append({ 'CIDR': '127.0.0.1/8', 'LastReport': None, 'UpdateStrategy': 'Disable', 'LastChange': None, 'MAC': '00:00:00:00:00:00', 'UpToDate': None, 'FQDN': None, 'DiskProfile': 'S', 'LastState': MachineStates.discovery, 'Roles': ScheduleRoles.kubernetes_control_plane }) ui = user_interface.UserInterfaceRepository(sess_maker=self.sess_maker) data = ui.get_machines_overview() self.assertCountEqual(expect, data)
def get_rolling_status(self, mac: str): with session_commit(sess_maker=self.sess_maker) as session: for m in session.query(Machine)\ .join(MachineInterface)\ .filter(MachineInterface.mac == mac)\ .join(LifecycleRolling): try: rolling = m.lifecycle_rolling[0] return rolling.enable, rolling.strategy except IndexError: pass logger.debug("mac: %s return None" % mac) return None, None
def get_machines_by_roles(self, *roles): if len(roles) == 1: return self.get_machines_by_role(roles[0]) machines = [] roles = list(roles) with session_commit(sess_maker=self.__sess_maker) as session: for machine in session.query(Machine): # TODO Maybe do this with a sqlalchemy filter func if set(k.role for k in machine.schedules) == set(roles): machines.append( self._construct_machine_dict(machine, roles)) return machines
def fetch(self, finished_in_less_than_min: int): time_limit = datetime.datetime.utcnow() - datetime.timedelta(minutes=finished_in_less_than_min) results = [] with session_commit(sess_maker=self.__sess_maker) as session: for machine in session.query(MachineCurrentState) \ .options(joinedload("interfaces")) \ .filter(MachineCurrentState.updated_date > time_limit) \ .order_by(MachineCurrentState.updated_date.desc()): results.append({ "fqdn": machine.interfaces[0].fqdn if machine.interfaces else None, "mac": machine.interfaces[0].mac if machine.interfaces else machine.machine_mac, "state": machine.state_name, "date": machine.updated_date }) return results
def test_no_machine_readd_disk_diff(self): mdr = MachineDiscoveryRepository(self.sess_maker) mdr.upsert(posts.M01) with session_commit(sess_maker=self.sess_maker) as session: self.assertEqual(1, session.query(Machine).count()) self.assertEqual(1, session.query(MachineInterface).count()) self.assertEqual(1, session.query(MachineDisk).count()) self.assertEqual(1, session.query(Chassis).count()) self.assertEqual(1, session.query(ChassisPort).count()) with_new_disk = copy.deepcopy(posts.M01) with_new_disk["disks"].append({ 'size-bytes': 21474836481, 'path': '/dev/sdb' }) mdr.upsert(with_new_disk) with session_commit(sess_maker=self.sess_maker) as session: self.assertEqual(1, session.query(Machine).count()) self.assertEqual(1, session.query(MachineInterface).count()) self.assertEqual(2, session.query(MachineDisk).count()) self.assertEqual(1, session.query(Chassis).count()) self.assertEqual(1, session.query(ChassisPort).count())
def test_one_machine_boot_interface_is_not_first(self): mac = "aa:bb:cc:dd:ee:0{}" with session_commit(sess_maker=self.sess_maker) as session: uuid = "b7f5f93a-b029-475f-b3a4-479ba198cb8a" machine = Machine(uuid=uuid) session.add(machine) session.flush() # add 3 non-boot interfaces for i in range(3): session.add( MachineInterface(machine_id=machine.id, mac=mac.format(i), netmask=1, ipv4="10.10.10.10", cidrv4="127.0.0.1/8", as_boot=False, gateway="1.1.1.1", name="foo")) # and a 4th one, with as_boot = True session.add( MachineInterface(machine_id=machine.id, mac=mac.format(3), netmask=1, ipv4="10.10.10.10", cidrv4="127.0.0.1/8", as_boot=True, gateway="1.1.1.1", name="foo")) session.commit() ms = MachineScheduleRepository(sess_maker=self.sess_maker) data = { "roles": ["kubernetes-control-plane", "etcd-member"], "selector": { "mac": mac.format(0) } } # schedule using the first but non-boot interface ms.create_schedule(data) s = ms.get_all_schedules() self.assertEqual(len(s), 1) # verify the scheduled machine is indexed by its boot-interface, the 3rd one in this case self.assertIn(mac.format(3), s)
def test_one_machine_scheduled_cp(self): mac = "00:00:00:00:00:00" with session_commit(sess_maker=self.sess_maker) as session: uuid = "b7f5f93a-b029-475f-b3a4-479ba198cb8a" machine = Machine(uuid=uuid) session.add(machine) machine_id = session.query(Machine).filter( Machine.uuid == uuid).first().id session.add( MachineInterface(machine_id=machine_id, mac=mac, netmask=1, ipv4="10.10.10.10", cidrv4="127.0.0.1/8", as_boot=True, gateway="1.1.1.1", name="lol")) session.add( Schedule(machine_id=machine_id, role=ScheduleRoles.etcd_member)) session.add( Schedule(machine_id=machine_id, role=ScheduleRoles.kubernetes_control_plane)) session.commit() ms = MachineScheduleRepository(sess_maker=self.sess_maker) ret = ms.get_available_machines() self.assertEqual(0, len(ret)) ret = ms.get_roles_by_mac_selector(mac) self.assertEqual([ ScheduleRoles.etcd_member, ScheduleRoles.kubernetes_control_plane ], ret) ret = ms.get_machines_by_roles(ScheduleRoles.etcd_member, ScheduleRoles.kubernetes_control_plane) self.assertEqual(1, len(ret)) ret = ms.get_machines_by_roles(ScheduleRoles.kubernetes_node) self.assertEqual(0, len(ret)) ret = ms.get_machines_by_roles(ScheduleRoles.etcd_member) self.assertEqual(1, len(ret)) ret = ms.get_machines_by_roles(ScheduleRoles.kubernetes_control_plane) self.assertEqual(1, len(ret))