def test_get_node_fewest_job(create_Jobs, create_Nodes): #create network #network1 = create_Network network6 = Network("fiat") assert network6.user == "fiat" assert network6.jobs == {} Job1, Job2, = list(create_Jobs(2)) Node1, Node2, = list(create_Nodes(2)) #test_function network6.add_job(Job1) network6.add_job(Job2) network6.add_node(Node1) network6.add_node(Node2) Node1.active = True #normally network.up(ip) for linux Node2.active = True #normally network.up(ip) for linux Node1.max_jobs = 4 Node2.max_jobs = 4 for job in JOBS.values(): #convert to GROUPS if isinstance(job.GROUPS, str): BaseGroup(job.GROUPS).add_job(job) elif isinstance(job.GROUPS, list): for g in job.GROUPS: BaseGroup(g).add_job(job) for node in NODES.values(): if isinstance(node.GROUPS, str): BaseGroup(node.GROUPS).add_node(node) elif isinstance(node.GROUPS, list): for g in node.GROUPS: BaseGroup(g).add_node(node) Node1.add_job(Job1) node = network6._get_node_fewest_jobs(Job2) assert node == Node2
def test_up(create_Nodes): #create network # network1 = create_Network network4 = Network("opel") assert network4.user == "opel" assert network4.jobs == {} Node1, = list(create_Nodes(1)) Node1.ip = "198.168.4.1" #for windows network4.add_node(Node1) #test function network4.up("198.168.4.1") assert Node1.active == True with pytest.raises(RuntimeError) as excinfo: network4.up("NodeNotExisting") assert f"Node with NodeNotExisting does not exist in Network database." in str(excinfo)
def test_is_ready(create_Jobs, create_Nodes): #create Network #network1 = create_Network network3 = Network("ttt") assert network3.user == "ttt" assert network3.jobs == {} Job1, Job2, = list(create_Jobs(2)) Node1, =list(create_Nodes(1)) network3.add_job(Job1) network3.add_job(Job2) network3.add_node(Node1) #check function Job1.allocate(Node1) assert network3.is_ready() == False Job2.allocate(Node1) assert network3.is_ready() == True
def test_down(create_Jobs, create_Nodes): #create network #network1 = create_Network network5 = Network("ford") assert network5.user == "ford" assert network5.jobs == {} Job1, Job2, = list(create_Jobs(2)) Node1, =list(create_Nodes(1)) Node1.ip = "198.168.4.1" #for windows Node1.max_jobs = 4 #for windows network5.add_job(Job1) network5.add_job(Job2) network5.add_node(Node1) Node1.add_job(Job2) #test function closed_jobs = network5.down("198.168.4.1") assert closed_jobs == [Job2] assert Node1.active == False with pytest.raises(RuntimeError) as excinfo: network5.down("NodeNotExisting") assert f"Node with NodeNotExisting does not exist in Network database." in str(excinfo)
def test_init_Network(): network1 = Network(user="******") assert network1.user == "pi" assert network1.jobs == {} assert network1.nodes == {} node = BaseNode("192.168.4.2") job = BaseJob() network2 = Network(user="******") assert network2.user == "asn" assert network2.jobs == {} assert network2.nodes == {} network2.add_job(job) assert network2.jobs == {1: job} network2.add_node(node) assert network2.nodes == {None : node} #fails for windows bc of dispy network3 = Network(user="******") assert network3.user == "super" assert network3.jobs == {} assert network3.nodes == {}
def test_add_job(create_Jobs): #create network #network1 = create_Network network2 = Network("pi") assert network2.user == "pi" assert network2.jobs == {} Job1, = list(create_Jobs(1)) #test with job object network2.add_job(Job1) assert Job1 in network2.jobs.values() #test with a non-job with pytest.raises(RuntimeError) as excinfo: network2.add_job("ThisIsNotAJob") assert f"Tried adding {str} as job to network." in str(excinfo)
def test_add_node(create_Nodes): #create network #network1 = create_Network network1 = Network("asn") assert network1.user == "asn" assert network1.jobs == {} Node1, = list(create_Nodes(1)) #test with Node object network1.add_node(Node1) assert Node1 in network1.nodes.values() #test with non Node object with pytest.raises(RuntimeError) as excinfo: network1.add_node("ThisIsNotANode") assert f"Tried adding {str} as node to network." in str(excinfo)
def test_preference(): #creating Network network1 = Network("pi") Node1 = BaseNode("192.168.4.1") Node2 = BaseNode("192.168.4.2") Node3 = BaseNode("192.168.4.3") Job1 = BaseJob(outputs=[BasePipe(1)]) Job2 = BaseJob(inputs=[BasePipe(1)],outputs=[BasePipe(2)]) Job3 = SquaredJob(inputs=[BasePipe(2)]) #--- Node1.ip = "192.168.4.1" Node2.ip = "192.168.4.2" Node3.ip = "192.168.4.3" #--- network1.add_job(Job1) network1.add_job(Job2) network1.add_job(Job3) network1.add_node(Node1) network1.add_node(Node2) network1.add_node(Node3) for job in JOBS.values(): #convert to GROUPS if isinstance(job.GROUPS, str): BaseGroup(job.GROUPS).add_job(job) elif isinstance(job.GROUPS, list): for g in job.GROUPS: BaseGroup(g).add_job(job) for node in NODES.values(): if isinstance(node.GROUPS, str): BaseGroup(node.GROUPS).add_node(node) elif isinstance(node.GROUPS, list): for g in node.GROUPS: BaseGroup(g).add_node(node) #Iinitalze Nodes #Node 1 and 2 will be alive, Node 3 will die #--- Node1.max_jobs = 4 Node2.max_jobs = 4 Node3.max_jobs = 4 #--- Node1.add_jobs([Job1]) Node2.add_jobs([Job2]) Node3.add_jobs([Job3]) network1.up("192.168.4.1") network1.up("192.168.4.2") network1.up("192.168.4.3") Node1.mac = "m1:b1:c1:z1" Node2.mac = "m2:b2:c2:z2" Node3.mac = "m3:b3:c3:z3" Node1.routing_table = {"mac": "m2:b2:c2:z2" , "TQ": 198} Node2.routing_table = {"mac": "m1:b1:c1:z1" , "TQ": 198} Node1._avg_info = {"cpu": 89, "memory": None, "disk": None, "swap": 0} Node2._avg_info = {"cpu": 91, "memory": None, "disk": None, "swap": 70} network1.down("192.168.4.3") node = network1.allocate_jobs(ssh= False) assert node == [Node2] assert len(Node2.jobs) == 2 assert len(Node3.jobs) == 0
def load_network(config, name=None, remote_root='', sanity_check=True, **kwargs): # load xml file if it is given if hasattr(config, "XML_FILE") and config.XML_FILE: if hasattr(config, "XML_ARGS"): xml_args = config.XML_ARGS else: xml_args = {} loader = XMLLoader(**kwargs) loader.load(config.XML_FILE) # load csv files if any loader = CSVLoader() csv_nodes = [n for n in NODES.values() if isinstance(n, str)] for path in csv_nodes: nodes = loader.load_nodes(path) print(f"Loaded {len(nodes)} nodes from {path}.") if config.CLIENT_IP: LocalNode(config.CLIENT_IP, name="LocalNode") else: LocalNode('localhost', name="LocalNode") # init groups for job in JOBS.values(): # check DEFAULT_NODE if job.DEFAULT_NODE: NODES[job.DEFAULT_NODE].add_job(job) # check LocalJob if isinstance(job, LocalJob): if hasattr(config, "CLIENT_IP") and config.CLIENT_IP: NODES[config.CLIENT_IP].add_job(job) else: NODES['localhost'].add_job(job) # convert to GROUPS if isinstance(job.GROUPS, str): BaseGroup(job.GROUPS).add_job(job) elif isinstance(job.GROUPS, list): for g in job.GROUPS: BaseGroup(g).add_job(job) # convert dependencies if job.DEPENDENCIES == None: pass elif isinstance(job.DEPENDENCIES, str): try: assert os.path.isdir(job.DEPENDENCIES) except AssertionError: raise RuntimeError( f"Dependencie {job.DEPENDENCIES} of {job} can`t be found or isn`t a directory." ) elif isinstance(job.DEPENDENCIES, list): deps = job.DEPENDENCIES job.DEPENDENCIES = None for dep in deps: if isinstance(dep, str): try: assert os.path.isdir(dep) except AssertionError: raise RuntimeError( f"Dependencie {dep} of {job} can`t be found or isn`t a directory." ) # check if dependencies already had been replaced if isinstance(job.DEPENDENCIES, str): raise RuntimeError( f"You may only define one path in DEPENDENCIES of {job}. {dep} is the second one." ) else: job.DEPENDENCIES = dep elif isinstance(dep, types.ModuleType) or inspect.isclass(dep): if hasattr(job, '_modules'): job._modules.append(dep) else: job._modules = [dep] else: raise RuntimeError( f"DEPENDENCIES of {job} may only contain modules or a path as a str" ) else: raise RuntimeError( f"DEPENDENCIES of job {job} must be str or list, not {type(job.DEPENDENCIES)}" ) for node in NODES.values(): if isinstance(node.GROUPS, str): BaseGroup(node.GROUPS).add_node(node) elif isinstance(node.GROUPS, list): for g in node.GROUPS: BaseGroup(g).add_node(node) if hasattr(config, 'DEBUG_WINDOW'): debug = config.DEBUG_WINDOW else: debug = False if hasattr(config, 'HEAD_SIZE'): if isinstance(config.HEAD_SIZE, int) and config.HEAD_SIZE >= 1: for pipe in PIPES.values(): pipe._head_size = config.HEAD_SIZE else: raise ValueError( f"HEAD_SIZE must be integer > 0, not {config.HEAD_SIZE}") if hasattr(config, "CLIENT_IP") and config.CLIENT_IP: client_ip = config.CLIENT_IP else: client_ip = "localhost" if hasattr(config, "PRE_COPY"): pre_copy = config.PRE_COPY else: pre_copy = False if sanity_check: for pipe in PIPES.values(): if not pipe.complete(): if not pipe.source: raise RuntimeError("{pipe} defines no source.") if not pipe.destination: raise RuntimeError("{pipe} defines no destination.") nodes = { ip: node for ip, node in NODES.items() if not isinstance(node, MultiNode) } network = Network(config.USER, JOBS, nodes, PIPES, debug=debug, name=name, client_ip=client_ip, remote_root=remote_root, pre_copy=pre_copy) return network