def test_run_alloc_cmd(self): os.chdir(basedir) config = {'tsi.testing': True} TSI.setup_defaults(config) # mock submit cmd config[ 'tsi.alloc_cmd'] = "echo 'salloc: Granted job allocation 115463'" cwd = os.getcwd() uspace = cwd + "/build/uspace-%s" % uuid.uuid4() os.mkdir(uspace) msg = """#!/bin/bash #TSI_SUBMIT #TSI_JOB_MODE allocate #TSI_USPACE_DIR %s #TSI_QUEUE fast #TSI_PROJECT myproject #TSI_NODES 4 """ % (uspace) control_out = io.StringIO() connector = MockConnector.MockConnector(None, control_out, None, None, self.LOG) self.bss.submit(msg, connector, config, self.LOG) result = control_out.getvalue() sleep(10) with open("%s/ALLOCATION_ID" % uspace) as f: line = f.readlines()[0] print("Allocation ID : %s" % line) self.assertTrue("115463" in line) os.chdir(cwd)
def test_submit_raw(self): config = {'tsi.testing': True} TSI.setup_defaults(config) # mock submit cmd config['tsi.submit_cmd'] = "echo 1234.server" cwd = os.getcwd() uspace = cwd + "/build/uspace-%s" % int(105 * time.time()) os.mkdir(uspace) msg = """#!/bin/bash #TSI_SUBMIT #TSI_JOB_MODE raw #TSI_JOB_FILE %s/tests/input/raw-job-file.sh #TSI_OUTCOME_DIR %s #TSI_USPACE_DIR %s """ % (cwd, uspace, uspace) control_source = io.BufferedReader(io.BytesIO(msg.encode("UTF-8"))) control_in = io.TextIOWrapper(control_source) control_out = io.StringIO() connector = MockConnector.MockConnector(control_in, control_out, None, None, self.LOG) self.bss.submit(msg, connector, config, self.LOG) result = control_out.getvalue() if "TSI_FAILED" in result: print(result) else: print("Submitted with ID %s" % result) print (uspace) control_source.close()
def test_submit_raw(self): os.chdir(basedir) config = {'tsi.testing': True} TSI.setup_defaults(config) # mock submit cmd config['tsi.submit_cmd'] = "echo 'Submitted batch job 1234'" cwd = os.getcwd() uspace = cwd + "/build/uspace-%s" % uuid.uuid4() os.mkdir(uspace) with open(uspace+"/foo.sh", "w") as f: f.write("""#!/bin/bash #SLURM --myopts """) msg = """#!/bin/bash #TSI_SUBMIT #TSI_JOB_MODE raw #TSI_JOB_FILE foo.sh #TSI_OUTCOME_DIR %s #TSI_USPACE_DIR %s ENDOFMESSAGE """ % (uspace, uspace) control_out = io.StringIO() connector = MockConnector.MockConnector(None, control_out, None, None, self.LOG) self.bss.submit(msg,connector, config, self.LOG) result = control_out.getvalue() assert "1234" in result os.chdir(cwd)
def test_submit_fail(self): os.chdir(basedir) config = {'tsi.testing': True} TSI.setup_defaults(config) # mock submit cmd config['tsi.submit_cmd'] = "/bin/false" cwd = os.getcwd() uspace = cwd + "/build/uspace-%s" % uuid.uuid4() os.mkdir(uspace) msg = """#!/bin/bash #TSI_SUBMIT #TSI_OUTCOME_DIR %s #TSI_USPACE_DIR %s ENDOFMESSAGE """ % (uspace, uspace) control_out = io.StringIO() connector = MockConnector.MockConnector(None, control_out, None, None, self.LOG) self.bss.submit(msg,connector, config, self.LOG) result = control_out.getvalue() print(result) assert "TSI_FAILED" in result os.chdir(cwd)
def test_submit_normal(self): os.chdir(basedir) config = {'tsi.testing': True} TSI.setup_defaults(config) # mock submit cmd config['tsi.submit_cmd'] = "echo 'Submitted batch job 1234'" cwd = os.getcwd() uspace = cwd + "/build/uspace-%s" % uuid.uuid4() os.mkdir(uspace) msg = """#!/bin/bash #TSI_SUBMIT #TSI_JOB_MODE normal #TSI_OUTCOME_DIR %s #TSI_USPACE_DIR %s #TSI_SCRIPT echo "Hello World!" ENDOFMESSAGE """ % (uspace, uspace) control_out = io.StringIO() connector = MockConnector.MockConnector(None, control_out, None, None, self.LOG) self.bss.submit(msg,connector, config, self.LOG) result = control_out.getvalue() assert "1234" in result os.chdir(cwd)
def test_setfacl_via_connector(self): acl_support = self.config['tsi.acl'] acl_support['/tmp/test_TSI_ACL'] = "POSIX" self.config['tsi.setfacl'] = "setfacl" user = "******" msg = """#TSI_FILE_ACL #TSI_ACL_OPERATION SETFACL #TSI_ACL_COMMAND MODIFY RECURSIVE #TSI_ACL_COMMAND_SPEC U %s rwx #TSI_ACL_PATH /tmp/test_TSI_ACL ENDOFMESSAGE """ % os.environ['USER'] connector = self.setup_connector(msg) TSI.process(connector, self.config, self.LOG) result = connector.control_out.getvalue() self.assertTrue("TSI_OK" in result) msg = """#TSI_FILE_ACL #TSI_ACL_OPERATION GETFACL #TSI_ACL_PATH /tmp/test_TSI_ACL ENDOFMESSAGE """ connector = self.setup_connector(msg) TSI.process(connector, self.config, self.LOG) result = connector.control_out.getvalue() self.assertTrue("TSI_OK" in result) print(result)
def test_get_file_chunk(self): cwd = os.getcwd() path = cwd + "/tests/input/testfile.txt" length = os.stat(path).st_size config = {'tsi.testing': True, 'tsi.switch_uid': False} msg = """#TSI_GETFILECHUNK #TSI_FILE %s #TSI_START 0 #TSI_LENGTH %d ENDOFMESSAGE """ % (path, length) control_source = io.BufferedReader(io.BytesIO(msg.encode("UTF-8"))) control_in = io.TextIOWrapper(control_source) control_out = io.StringIO() data_out = io.BytesIO() connector = MockConnector.MockConnector(control_in, control_out, None, data_out, self.LOG) TSI.process(connector, config, self.LOG) result = control_out.getvalue() data = str(data_out.getvalue()) self.assertTrue("TSI_OK" in result) self.assertTrue("this is a test file" in data) control_source.close() data_out.close() os.chdir(cwd)
def setUp(self): # setup logger self.LOG = Log.Logger("tsi.testing") self.config = {'tsi.testing': True, 'tsi.switch_uid': False} TSI.setup_defaults(self.config) self.bss = BSS.BSS() self.bss.init(self.config, self.LOG)
def test_submit(self): config = {'tsi.testing': True} TSI.setup_defaults(config) # mock submit cmd config['tsi.submit_cmd'] = "echo 1234.server" cwd = os.getcwd() uspace = cwd + "/build/uspace-%s" % int(100 * time.time()) os.mkdir(uspace) msg = """#!/bin/bash #TSI_UFTP #TSI_USPACE_DIR %s #TSI_OUTCOME_DIR %s #TSI_UFTP_HOST localhost #TSI_UFTP_PORT 54434 #TSI_UFTP_SECRET test123 #TSI_UFTP_MODE GET #TSI_UFTP_REMOTE_FILE foo #TSI_UFTP_LOCAL_FILE bar """ % (uspace, uspace) control_source = io.BufferedReader(io.BytesIO(msg.encode("UTF-8"))) control_in = io.TextIOWrapper(control_source) control_out = io.StringIO() connector = MockConnector.MockConnector(control_in, control_out, None, None, self.LOG) UFTP.uftp(msg, connector, config, self.LOG) result = control_out.getvalue() if "TSI_FAILED" in result: print(result) else: print("OK!") control_source.close() os.chdir(cwd)
def setUp(self): self.LOG = Log.Logger("tsi.testing") self.bss = lsf.BSS.BSS() self.config = {'tsi.testing': True, # mock submit cmd 'tsi.submit_cmd': "echo 1234.server" } TSI.setup_defaults(self.config) self.bss.init(self.config, self.LOG)
def test_parse_details(self): os.chdir(basedir) config = {'tsi.testing': True} TSI.setup_defaults(config) with open("tests/input/details_slurm.txt", "r") as f: raw = f.read() parsed = self.bss.parse_job_details(raw) print(parsed)
def setUp(self): self.LOG = Log.Logger("tsi.testing") self.bss = slurm.BSS.BSS() self.config = { 'tsi.testing': True, # mock submit/alloc cmds 'tsi.submit_cmd': "echo 'Submitted batch job 1234'", 'tsi.alloc_cmd': "echo 'salloc: Granted job allocation 115463'" } TSI.setup_defaults(self.config) self.bss.init(self.config, self.LOG)
def test_report_details(self): os.chdir(basedir) config = {'tsi.testing': True} config['tsi.details_cmd'] = "cat " TSI.setup_defaults(config) control_out = io.StringIO() connector = MockConnector.MockConnector(None, control_out, None, None, self.LOG) msg = "#TSI_BSSID tests/input/details_slurm.txt\n" self.bss.get_job_details(msg, connector, config, self.LOG) result = control_out.getvalue() print(result)
def test_submit(self): cwd = os.getcwd() uspace = cwd + "/build/uspace-%s" % int(time.time()) os.mkdir(uspace) config = {'tsi.testing': True, 'tsi.switch_uid': False} bss = BSS.BSS() bss.init(config, self.LOG) msg = """#!/bin/bash #TSI_SUBMIT #TSI_OUTCOME_DIR %s #TSI_USPACE_DIR %s #TSI_STDOUT stdout #TSI_STDERR stderr #TSI_SCRIPT #TSI_TIME 60 #TSI_MEMORY 32 #TSI_JOBNAME test_job #TSI_SCRIPT echo "Hello World!" sleep 1 ENDOFMESSAGE """ % (uspace, uspace) control_source = io.BufferedReader(io.BytesIO(msg.encode("UTF-8"))) control_in = io.TextIOWrapper(control_source) control_out = io.StringIO() connector = MockConnector.MockConnector(control_in, control_out, None, None, self.LOG) TSI.process(connector, config, self.LOG) result = control_out.getvalue() if "TSI_FAILED" in result: print(result) else: result = result.splitlines()[0] print("Submitted with ID %s" % result) control_source.close() children = config.get('tsi.NOBATCH.children') print("Children: " + str(children)) self.assertEqual(1, len(children)) control_out = io.StringIO() connector = MockConnector.MockConnector(control_in, control_out, None, None, self.LOG) bss.get_status_listing(None, connector, config, self.LOG) qstat = control_out.getvalue() print(qstat + "\n") self.assertTrue(result in qstat) # test cleanup time.sleep(2) bss.cleanup(config) print("Children after cleanup: " + str(children)) self.assertEqual(0, len(children))
def test_put_file_chunk(self): cwd = os.getcwd() path = cwd + "/build/testfile.txt" config = {'tsi.testing': True, 'tsi.switch_uid': False} data = b"this is some testdata used for testing the TSI I/O \n" msg = """#TSI_PUTFILECHUNK #TSI_FILE %s 600 #TSI_FILESACTION 1 #TSI_START 0 #TSI_LENGTH %d ENDOFMESSAGE """ % (path, len(data)) control_source = io.BufferedReader(io.BytesIO(msg.encode("UTF-8"))) control_in = io.TextIOWrapper(control_source) control_out = io.StringIO() data_in = io.BytesIO(data) connector = MockConnector.MockConnector(control_in, control_out, data_in, None, self.LOG) TSI.process(connector, config, self.LOG) result = control_out.getvalue() print(result) self.assertTrue("TSI_OK" in result) control_source.close() lines = self.readlines(path) self.assertEqual(1, len(lines)) self.assertTrue(data.decode() in lines[0]) # test append msg = """#TSI_PUTFILECHUNK #TSI_FILE %s 600 #TSI_FILESACTION 3 #TSI_START 0 #TSI_LENGTH %d ENDOFMESSAGE """ % (path, len(data)) control_source = io.BufferedReader(io.BytesIO(msg.encode("UTF-8"))) control_in = io.TextIOWrapper(control_source) control_out = io.StringIO() data_in = io.BytesIO(data) connector = MockConnector.MockConnector(control_in, control_out, data_in, None, self.LOG) TSI.process(connector, config, self.LOG) result = control_out.getvalue() print(result) self.assertTrue("TSI_OK" in result) lines = self.readlines(path) self.assertEqual(2, len(lines)) self.assertTrue(data.decode() in lines[0]) os.chdir(cwd)
def test_getsupport_via_connector(self): acl_support = self.config['tsi.acl'] acl_support['/tmp'] = "POSIX" msg = """#TSI_FILE_ACL #TSI_ACL_OPERATION CHECK_SUPPORT #TSI_ACL_PATH /tmp ENDOFMESSAGE """ connector = self.setup_connector(msg) TSI.process(connector, self.config, self.LOG) result = connector.control_out.getvalue() self.assertTrue("TSI_OK" in result) self.assertTrue("true" in result)
def setUp(self): # setup logger self.LOG = logging.getLogger("tsi.testing") self.LOG.setLevel(logging.INFO) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) self.LOG.handlers = [ch] self.config = {'tsi.testing': True} TSI.setup_defaults(self.config) BecomeUser.initialize(self.config, self.LOG)
def test_RunMain(self): config = "tests/conf/tsi.properties" pid = os.fork() if pid == 0: # child, this is the TSI shepherd process TSI.main(["TSI", config]) else: # parent, this is the fake XNJS LOG = logging.getLogger("fake-xnjs") LOG.setLevel(logging.INFO) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) LOG.handlers = [ch] time.sleep(2) client_config = TSI.read_config_file(config, LOG) # connect to the server host = client_config['tsi.my_addr'] port = int(client_config['tsi.my_port']) tsi = socket.create_connection((host, port)) LOG.info("CLIENT: Connected to %s:%s" % (host, port)) host = client_config['tsi.njs_machine'] port = int(client_config['tsi.njs_port']) tsi.sendall(b'newtsiprocess 24433') LOG.info("CLIENT: waiting for callback on %s:%s" % (host, port)) server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server.bind((host, port)) server.listen(2) (command, (_, _)) = server.accept() (data, (_, _)) = server.accept() test_msg = b'#TSI_PING\nENDOFMESSAGE\n' LOG.info("CLIENT: connected, sending test message: %s" % test_msg) command.sendall(test_msg) reply = command.recv(1024) print(reply) # send shutdown and cleanup LOG.info("CLIENT: shutdown") tsi.sendall(b'shutdown') command.close() data.close() tsi.close() server.close() os.kill(pid, signal.SIGKILL)
def test_PING(self): cwd = os.getcwd() version = TSI.MY_VERSION config = {'tsi.testing': True} msg = "#TSI_PING\nENDOFMESSAGE\n" control_source = io.BufferedReader(io.BytesIO(msg.encode("UTF-8"))) control_in = io.TextIOWrapper(control_source) control_out = io.StringIO() connector = MockConnector.MockConnector(control_in, control_out, None, None, self.LOG) TSI.process(connector, config, self.LOG) result = control_out.getvalue() print(result) self.assertTrue(version in result) self.assertTrue("ENDOFMESSAGE" in result) control_source.close() os.chdir(cwd)
def test_Exec(self): cwd = os.getcwd() config = {'tsi.testing': True, 'tsi.switch_uid': False} msg = """#TSI_EXECUTESCRIPT echo "Hello World!" ENDOFMESSAGE """ control_source = io.BufferedReader(io.BytesIO(msg.encode("UTF-8"))) control_in = io.TextIOWrapper(control_source) control_out = io.StringIO() connector = MockConnector.MockConnector(control_in, control_out, None, None, self.LOG) TSI.process(connector, config, self.LOG) result = control_out.getvalue() print(result) self.assertTrue("TSI_OK" in result) self.assertTrue("Hello World!\n" in result) control_source.close() os.chdir(cwd)
def test_PING2(self): cwd = os.getcwd() version = TSI.MY_VERSION config = {'tsi.testing': True, 'tsi.enforce_os_gids': False} config['tsi.use_id_to_resolve_gids'] = False msg = """#TSI_PING_UID #TSI_IDENTITY nobody NONE ENDOFMESSAGE """ control_source = io.BufferedReader(io.BytesIO(msg.encode("UTF-8"))) control_in = io.TextIOWrapper(control_source) control_out = io.StringIO() BecomeUser.initialize(config, self.LOG) connector = MockConnector.MockConnector(control_in, control_out, None, None, self.LOG) TSI.process(connector, config, self.LOG) result = control_out.getvalue() print(result) self.assertTrue(version in result) control_source.close() os.chdir(cwd)
def test_create_submit_script(self): os.chdir(basedir) config = {'tsi.testing': True} TSI.setup_defaults(config) # mock submit cmd config['tsi.submit_cmd'] = "echo 'Submitted batch job 1234'" cwd = os.getcwd() uspace = cwd + "/build/uspace-%s" % uuid.uuid4() os.mkdir(uspace) msg = """#!/bin/bash #TSI_SUBMIT #TSI_OUTCOME_DIR %s #TSI_USPACE_DIR %s #TSI_STDOUT stdout #TSI_STDERR stderr #TSI_SCRIPT #TSI_QUEUE fast #TSI_PROJECT myproject #TSI_TIME 60 #TSI_MEMORY 32 #TSI_NODES 1 #TSI_PROCESSORS_PER_NODE 64 #TSI_ARRAY 10 #TSI_ARRAY_LIMIT 2 #TSI_BSS_NODES_FILTER NONE #TSI_JOBNAME test_job #TSI_SCRIPT echo "Hello World!" sleep 3 ENDOFMESSAGE """ % (uspace, uspace) submit_cmds = self.bss.create_submit_script(msg, config, self.LOG) self.assertTrue(self.has_directive(submit_cmds, "#SBATCH --partition", "fast")) self.assertTrue(self.has_directive(submit_cmds, "#SBATCH --nodes", "1")) self.assertTrue(self.has_directive(submit_cmds, "#SBATCH --ntasks-per-node", "64")) self.assertTrue(self.has_directive(submit_cmds, "#SBATCH --mem", "32")) self.assertTrue(self.has_directive(submit_cmds, "#SBATCH --time", "1")) self.assertTrue(self.has_directive(submit_cmds, "#SBATCH --array", "10%2")) self.assertTrue(self.has_directive(submit_cmds, "#SBATCH --account", "myproject")) self.assertFalse(self.has_directive(submit_cmds, "#SBATCH --constraint"))
def test_submit(self): config = {'tsi.testing': True} TSI.setup_defaults(config) # mock submit cmd config['tsi.submit_cmd'] = "echo 1234.server" cwd = os.getcwd() uspace = cwd + "/build/uspace-%s" % int(100 * time.time()) os.mkdir(uspace) msg = """#!/bin/bash #TSI_SUBMIT #TSI_OUTCOME_DIR %s #TSI_USPACE_DIR %s #TSI_STDOUT stdout #TSI_STDERR stderr #TSI_SCRIPT #TSI_QUEUE fast #TSI_PROJECT myproject #TSI_TIME 60 #TSI_MEMORY 32 #TSI_NODES 1 #TSI_PROCESSORS_PER_NODE 64 #TSI_ARRAY 10 #TSI_ARRAY_LIMIT 2 #TSI_JOBNAME test_job #TSI_SCRIPT echo "Hello World!" sleep 3 """ % (uspace, uspace) submit_cmds = self.bss.create_submit_script(msg, config, self.LOG) print(submit_cmds) self.assertTrue(self.has_directive(submit_cmds, "#PBS -q", "fast")) self.assertTrue( self.has_directive(submit_cmds, "#PBS -l", "nodes=1:ppn=64")) self.assertTrue( self.has_directive(submit_cmds, "#PBS -l", "walltime=60")) self.assertTrue(self.has_directive(submit_cmds, "#PBS -A", "myproject")) self.assertTrue(self.has_directive(submit_cmds, "#PBS -t", "10%2"))
def test_getfacl_via_connector(self): acl_support = self.config['tsi.acl'] acl_support['/tmp'] = "POSIX" msg = """#TSI_FILE_ACL #TSI_ACL_OPERATION GETFACL #TSI_ACL_PATH /tmp ENDOFMESSAGE """ connector = self.setup_connector(msg) TSI.process(connector, self.config, self.LOG) result = connector.control_out.getvalue() self.assertTrue("TSI_OK" in result) msg = """#TSI_FILE_ACL #TSI_ACL_OPERATION GETFACL #TSI_ACL_PATH /opt ENDOFMESSAGE """ connector = self.setup_connector(msg) TSI.process(connector, self.config, self.LOG) result = connector.control_out.getvalue() print(result) self.assertTrue("TSI_FAILED" in result) self.assertTrue("unsupported" in result)
def test_read_config(self): cwd = os.getcwd() file = "tests/input/test_config.properties" c = TSI.read_config_file(file, self.LOG) # parse allowed DNs correctly? acl = c["tsi.allowed_dns"] from lib import SSL subject = ((('commonName', 'Some Guy'),), (('countryName','EU',),)) self.assertTrue(SSL.match(subject, acl), msg="should match %s" % str(subject)) subject = ((('commonName', 'Some Guy'),), (('countryName','DE',),)) self.assertFalse(SSL.match(subject, acl), msg="wrong match %s" % str(subject)) # accept white space in property lines? self.assertEqual("some_value", c["whitespace"]) os.chdir(cwd)
def test_version_check(self): version_ok = TSI.assert_version() self.assertTrue(version_ok)
def setUp(self): self.LOG = Log.Logger("tsi.testing", use_syslog=False) self.config = {'tsi.testing': True, "tsi.switch_uid": False} TSI.setup_defaults(self.config) BecomeUser.initialize(self.config, self.LOG)
def test_configure(self): config = TSI.read_config_file(self.file_name, self.LOG) self.assertEqual('600', config['tsi.usersCacheTtl']) acl = config['tsi.acl'] self.assertEqual('NONE', acl['/'])