def test_03_create_user_proxy(self): # if the utils are not present, it won't work anyhow, so might as well skip the test if not core.rpm_is_installed('globus-proxy-utils'): core.skip('globus-proxy-utils not installed') return # ...and also skip if there is no glexec if not core.rpm_is_installed('glexec'): core.skip('not installed') return # OK, software is present, now just check it previous tests did create the proxy already so # we don't do it twice command = ('grid-proxy-info','-f',self.__user_proxy_path) status, stdout, stderr = core.system(command, True) if int(status)!=0: # no proxy found for some reason, try to construct a new one command = ('grid-proxy-init','-out',self.__user_proxy_path) password = core.options.password + '\n' status, stdout, stderr = core.system(command, True, password) self.assert_(status==0, 'grid-proxy-init for user '+core.options.username+' has failed even though globus-proxy-util was present') # we need to have the right permissions on that proxy for glexec to agree to work, # and the easiest way is to copy the file command = ('/bin/cp', self.__user_proxy_path, self.__glexec_client_cert) status, stdout, stderr = core.system(command) os.environ['GLEXEC_CLIENT_CERT']=self.__glexec_client_cert
def test_01_create_macaroons(self): core.config['xrootd.tpc.macaroon-1'] = None core.config['xrootd.tpc.macaroon-2'] = None core.skip_ok_unless_installed('x509-scitokens-issuer-client', by_dependency=True) self.skip_bad_unless(core.state['proxy.valid'], 'requires a proxy cert') uid = pwd.getpwnam(core.options.username)[2] usercert = '/tmp/x509up_u%d' % uid userkey = '/tmp/x509up_u%d' % uid core.config['xrootd.tpc.url-1'] = "https://" + core.get_hostname() + ":9001" + "/usr/share/osg-test/test_gridftp_data.txt".strip() command = ('macaroon-init', core.config['xrootd.tpc.url-1'], '20', 'ALL') status, stdout, stderr = core.system(command, user=True) fail = core.diagnose('Obtain Macaroon 1', command, status, stdout, stderr) self.assertEqual(status, 0, fail) core.config['xrootd.tpc.macaroon-1'] = stdout.strip() core.config['xrootd.tpc.url-2'] = "https://" + core.get_hostname() + ":9002" + "/tmp/test_gridftp_data_tpc.txt".strip() command = ('macaroon-init', core.config['xrootd.tpc.url-2'], '20', 'ALL') status, stdout, stderr = core.system(command, user=True) fail = core.diagnose('Obtain Macaroon 2', command, status, stdout, stderr) self.assertEqual(status, 0, fail) core.config['xrootd.tpc.macaroon-2'] = stdout.strip()
def test_01_cvmfs_probe(self): default_local = '/etc/cvmfs/default.local' probe_repos = ",".join([ 'atlas.cern.ch', 'cms.cern.ch', 'oasis.opensciencegrid.org']) # Test depends on oasis-config to access the oasis.opensciencegrid.org # repo. This is an external service, so the requirement should be # removed as part of SOFTWARE-1108. core.skip_ok_unless_installed('cvmfs') core.skip_ok_unless_installed('cvmfs-keys', 'oasis-config', by_dependency=True) command = ('cat', default_local) status, stdout, stderr = core.system(command, False) # Dave Dykstra suggested running cvmfs probe against a different # set of repositories than are currently set up, so we modify them # just for this test. (See SOFTWARE-1097) # In the future, this test might be removed since we do not want # to depend on external services, and it's redundant to probe the # repos that we have already mounted. files.replace( default_local, 'CVMFS_REPOSITORIES=cms.cern.ch', 'CVMFS_REPOSITORIES=' + probe_repos, owner='cvmfsprobe') try: command = ('cvmfs_config', 'probe') status, stdout, stderr = core.system(command, False) self.assertEqual(status, 0, core.diagnose('cvmfs probe', command, status, stdout, stderr)) finally: files.restore(default_local, 'cvmfsprobe')
def run_trace(self, *args): """Run condor_ce_trace along with any additional *args. If trace completes with a held job, also return output from 'condor_ce_q -held'. """ cwd = os.getcwd() os.chdir('/tmp') self.command += ['condor_ce_trace', '--debug' ] + list(args) + [core.get_hostname()] if core.osg_release() == "3.6" and \ core.PackageVersion('condor') >= '9.0.0' and \ core.PackageVersion('condor') < '9.0.8': with core.no_x509(core.options.username): trace_rc, trace_out, trace_err = core.system(self.command, user=True) else: trace_rc, trace_out, trace_err = core.system(self.command, user=True) os.chdir(cwd) if trace_rc: msg = 'condor_ce_trace failed' if trace_out.find(', was held'): msg = 'condor_ce_trace job held' _, hold_out, hold_err = core.system(('condor_ce_q', '-held')) self.fail( core.diagnose(msg, self.command, trace_rc, str(trace_out) + str(hold_out), str(trace_err) + str(hold_err))) return trace_out, trace_err
def test_15_checkdatabase_dcache_storage(self): core.skip_ok_unless_installed('gratia-probe-dcache-storage', 'gratia-service') self.skip_ok_if(core.state['gratia.dcache-whitelisted-error'], 'caught whitelisted error') self.skip_bad_unless(core.state['gratia.dcache-storage-running']) self.assertEqual(True, self.isProbeInfoProcessed('dCache-storage'), 'Sentinel signifying that Probe Information was processed NOT found.') command = "echo \"use gratia_osgtest; " + \ "select TotalSpace from StorageElementRecord where ProbeName like 'dCache-storage%';" + \ core.config['gratia.sql.querystring'], status, total_space, _ = core.system(command, shell=True) self.assertEqual(status, 0, 'Unable to query Gratia Database TotalSpace from StorageElementRecord table.') command = "echo \"use gratia_osgtest; " + \ "select FreeSpace from StorageElementRecord where ProbeName like 'dCache-storage%';" + \ core.config['gratia.sql.querystring'], status, free_space, _ = core.system(command, shell=True) self.assertEqual(status, 0, 'Unable to query Gratia Database FreeSpace from StorageElementRecord table.') command = "echo \"use gratia_osgtest; " + \ "select UsedSpace from StorageElementRecord where ProbeName like 'dCache-storage%';" + \ core.config['gratia.sql.querystring'], status, used_space, _ = core.system(command, shell=True) self.assertEqual(status, 0, 'Unable to query Gratia Database UsedSpace from StorageElementRecord table.') #Need to assert only after converting string to long... self.assert_(long(total_space) == (long(free_space) + long(used_space)))
def test_04_xrootd_fuse(self): # This tests xrootd-fuse using a mount in /mnt core.skip_ok_unless_installed('xrootd', 'xrootd-client', by_dependency=True) self.skip_ok_unless(os.path.exists("/mnt"), "/mnt did not exist") self.skip_ok_if(core.config['xrootd.gsi'] == "ON", 'fuse incompatible with GSI') if not os.path.exists(TestXrootd.__fuse_path): os.mkdir(TestXrootd.__fuse_path) hostname = socket.getfqdn() #command = ('xrootdfs',TestXrootd.__fuse_path,'-o','rdr=xroot://localhost:1094//tmp','-o','uid=xrootd') command = ('mount', '-t', 'fuse', '-o', 'rdr=xroot://localhost:1094//tmp,uid=xrootd', 'xrootdfs', TestXrootd.__fuse_path) command_str = ' '.join(command) #For some reason, sub process hangs on fuse processes, use os.system #status, stdout, stderr = core.system(command_str,shell=True) os.system(command_str) # Copy a file in and see if it made it into the fuse mount xrootd_url = 'root://%s/%s/copied_file.txt' % (hostname, "/tmp") command = ('xrdcp', '--debug', '3', TestXrootd.__data_path, xrootd_url) core.system(command, user=True) command = ('ls', "/tmp/copied_file.txt") core.check_system(command, "Checking file is copied to xrootd fuse mount correctly", user=True) command = ('umount', TestXrootd.__fuse_path) core.system(command) os.rmdir(TestXrootd.__fuse_path) files.remove("/tmp/copied_file.txt")
def test_01_cvmfs_probe(self): default_local = '/etc/cvmfs/default.local' probe_repos = ",".join( ['atlas.cern.ch', 'cms.cern.ch', 'oasis.opensciencegrid.org']) # Test depends on oasis-config to access the oasis.opensciencegrid.org # repo. This is an external service, so the requirement should be # removed as part of SOFTWARE-1108. core.skip_ok_unless_installed('cvmfs') core.skip_ok_unless_installed('cvmfs-keys', 'oasis-config', by_dependency=True) command = ('cat', default_local) status, stdout, stderr = core.system(command, False) # Dave Dykstra suggested running cvmfs probe against a different # set of repositories than are currently set up, so we modify them # just for this test. (See SOFTWARE-1097) # In the future, this test might be removed since we do not want # to depend on external services, and it's redundant to probe the # repos that we have already mounted. files.replace(default_local, 'CVMFS_REPOSITORIES=cms.cern.ch', 'CVMFS_REPOSITORIES=' + probe_repos, owner='cvmfsprobe') try: command = ('cvmfs_config', 'probe') status, stdout, stderr = core.system(command, False) self.assertEqual( status, 0, core.diagnose('cvmfs probe', command, status, stdout, stderr)) finally: files.restore(default_local, 'cvmfsprobe')
def setup_cvmfs(): command = ('mkdir', '-p', '/tmp/cvmfs') core.system(command, False) files.write("/etc/cvmfs/default.local", CVMFS_CONFIG, owner='cvmfs', chmod=0o644) # Dump autofs debug output to /var/log/messages or journalctl files.append("/etc/sysconfig/autofs", 'OPTIONS="-d"\n', owner='cvmfs')
def test_04_xrootd_fuse(self): # This tests xrootd-fuse using a mount in /mnt core.skip_ok_unless_installed('xrootd', 'xrootd-client', 'xrootd-fuse') self.skip_ok_unless(os.path.exists("/mnt"), "/mnt did not exist") if not os.path.exists(TestXrootd.__fuse_path): os.mkdir(TestXrootd.__fuse_path) hostname = socket.getfqdn() # For some reason, sub process hangs on fuse processes, use os.system os.system( "mount -t fuse -o rdr=root://localhost//tmp,uid=xrootd xrootdfs %s" % TestXrootd.__fuse_path) # Copy a file in and see if it made it into the fuse mount xrootd_url = 'root://%s/%s/copied_file.txt' % (hostname, "/tmp") core.system( ['xrdcp', '--debug', '3', TestXrootd.__data_path, xrootd_url], user=True) self.assert_(os.path.isfile("/tmp/copied_file.txt"), "Test file not uploaded to FUSE mount") core.system(['umount', TestXrootd.__fuse_path]) os.rmdir(TestXrootd.__fuse_path) files.remove("/tmp/copied_file.txt")
def test_03_xrootd_fuse(self): """ This tests xrootd-fuse using a mount in /mnt """ if core.missing_rpm('xrootd-server', 'xrootd-client','xrootd-fuse'): return if not os.path.exists("/mnt"): core.log_message("/mnt did not exist, skipping xrootd fuse test") return if not os.path.exists(TestXrootd.__fuse_path): os.mkdir(TestXrootd.__fuse_path) if core.config['xrootd.gsi'] == "ON": core.log_message("fuse incompatible with GSI, skipping xrootd fuse") return hostname = socket.getfqdn() #command = ('xrootdfs',TestXrootd.__fuse_path,'-o','rdr=xroot://localhost:1094//tmp','-o','uid=xrootd') command = ('mount', '-t','fuse','-o','rdr=xroot://localhost:1094//tmp,uid=xrootd','xrootdfs',TestXrootd.__fuse_path) command_str= ' '.join(command) #For some reason, sub process hangs on fuse processes, use os.system #status, stdout, stderr = core.system(command_str,shell=True) os.system(command_str) # Copy a file in and see if it made it into the fuse mount xrootd_url = 'root://%s/%s/copied_file.txt' % (hostname, "/tmp") command = ('xrdcp', TestXrootd.__data_path , xrootd_url) status, stdout, stderr = core.system(command, True) command = ('ls', "/tmp/copied_file.txt") stdout, stderr, fail = core.check_system(command, "Checking file is copied to xrootd fuse mount correctly", user=True) command = ('umount',TestXrootd.__fuse_path) status, stdout, stderr = core.system(command) os.rmdir(TestXrootd.__fuse_path) files.remove("/tmp/copied_file.txt")
def test_02_cvmfs(self): core.skip_ok_unless_installed('cvmfs') core.skip_ok_unless_installed('cvmfs-keys', by_dependency=True) core.state['cvmfs.mounted'] = False command = ('ls', '/cvmfs') status, stdout, stderr = core.system(command, False) file_exists = os.path.exists('/cvmfs') self.assert_(file_exists, 'Cvmfs mount point missing') core.state['cvmfs.mounted'] = True cern_repo = 'cms.cern.ch' command = ('ls', '/cvmfs/' + cern_repo) status, stdout, stderr = core.system(command, False) # If the previous command failed, output better debug info if status != 0: self.debug_cvmfs(cern_repo) command = ('ls', self.__check_path) status, stdout, stderr = core.system(command, False) self.assert_(file_exists, 'Test cvmfs file missing') command = ('bash', '-c', 'source ' + self.__check_path) status, stdout, stderr = core.system(command, False) fail = core.diagnose('cvmfs example source a file on fs', command, status, stdout, stderr) self.assertEqual(status, 0, fail)
def test_01_configure_xrootd(self): core.state['xrootd.is-configured'] = False core.config['xrootd.security'] = set() core.config['certs.xrootdcert'] = '/etc/grid-security/xrd/xrdcert.pem' core.config['certs.xrootdkey'] = '/etc/grid-security/xrd/xrdkey.pem' # rootdir and resourcename needs to be set early for the default osg-xrootd config core.config['xrootd.config'] = '/etc/xrootd/config.d/10-osg-test.cfg' core.config['xrootd.logging-config'] = '/etc/xrootd/config.d/99-logging.cfg' core.config['xrootd.service-defaults'] = '/etc/sysconfig/xrootd' core.config['xrootd.multiuser'] = False core.state['xrootd.backups-exist'] = False core.state['xrootd.had-failures'] = False core.config['xrootd.public_subdir'] = "public" core.config['xrootd.user_subdir'] = core.options.username core.config['xrootd.vo_subdir'] = voms.VONAME core.config['xrootd.authfile'] = '/etc/xrootd/Authfile' self.skip_ok_unless(core.state['user.verified'], "Test user not available") xrootd_user = pwd.getpwnam("xrootd") xrootd_config = STANDALONE_XROOTD_CFG_TEXT if core.dependency_is_installed("voms-clients"): core.config['xrootd.security'].add("GSI") if core.PackageVersion("xrootd-scitokens") >= "5": core.config['xrootd.security'].add("SCITOKENS") if voms.can_make_proxy(): core.config['xrootd.security'].add("VOMS") self.skip_ok_unless(core.config['xrootd.security'], "No xrootd security available") core.install_cert('certs.xrootdcert', 'certs.hostcert', 'xrootd', 0o644) core.install_cert('certs.xrootdkey', 'certs.hostkey', 'xrootd', 0o400) files.write(core.config['xrootd.logging-config'], XROOTD_LOGGING_CFG_TEXT, owner='xrootd', backup=True, chmod=0o644) files.write(core.config['xrootd.config'], xrootd_config, owner='xrootd', backup=True, chmod=0o644) files.write(core.config['xrootd.authfile'], AUTHFILE_TEXT, owner="xrootd", chown=(xrootd_user.pw_uid, xrootd_user.pw_gid), chmod=0o644) try: shutil.rmtree(xrootd.ROOTDIR) except FileNotFoundError: pass public_dir = f"{xrootd.ROOTDIR}/{core.config['xrootd.public_subdir']}" files.safe_makedirs(xrootd.ROOTDIR) os.chmod(xrootd.ROOTDIR, 0o755) files.safe_makedirs(public_dir) os.chmod(public_dir, 0o1777) user_dir = f"{xrootd.ROOTDIR}/{core.config['xrootd.user_subdir']}" files.safe_makedirs(user_dir) os.chmod(user_dir, 0o770) vo_dir = f"{xrootd.ROOTDIR}/{core.config['xrootd.vo_subdir']}" files.safe_makedirs(vo_dir) os.chmod(vo_dir, 0o1777) core.system(["chown", "-R", "xrootd:xrootd", xrootd.ROOTDIR]) os.chown(user_dir, core.state["user.uid"], xrootd_user.pw_gid) core.check_system(["find", xrootd.ROOTDIR, "-ls"], f"Couldn't dump contents of {xrootd.ROOTDIR}") core.state['xrootd.backups-exist'] = True core.state['xrootd.is-configured'] = True
def test_03_xrootd_fuse(self): # This tests xrootd-fuse using a mount in /mnt core.skip_ok_unless_installed('xrootd', 'xrootd-client', by_dependency=True) self.skip_ok_unless(os.path.exists("/mnt"), "/mnt did not exist") self.skip_ok_if(core.config['xrootd.gsi'] == "ON", 'fuse incompatible with GSI') if not os.path.exists(TestXrootd.__fuse_path): os.mkdir(TestXrootd.__fuse_path) hostname = socket.getfqdn() #command = ('xrootdfs',TestXrootd.__fuse_path,'-o','rdr=xroot://localhost:1094//tmp','-o','uid=xrootd') command = ('mount', '-t', 'fuse', '-o', 'rdr=xroot://localhost:1094//tmp,uid=xrootd', 'xrootdfs', TestXrootd.__fuse_path) command_str = ' '.join(command) #For some reason, sub process hangs on fuse processes, use os.system #status, stdout, stderr = core.system(command_str,shell=True) os.system(command_str) # Copy a file in and see if it made it into the fuse mount xrootd_url = 'root://%s/%s/copied_file.txt' % (hostname, "/tmp") command = ('xrdcp', '--debug', '3', TestXrootd.__data_path, xrootd_url) core.system(command, user=True) command = ('ls', "/tmp/copied_file.txt") core.check_system(command, "Checking file is copied to xrootd fuse mount correctly", user=True) command = ('umount', TestXrootd.__fuse_path) core.system(command) os.rmdir(TestXrootd.__fuse_path) files.remove("/tmp/copied_file.txt")
def clean(*repos): """Perform 'yum clean' commands that we recommend to our users""" if not repos: # clean all repos if none specified repos = ['*'] core.system(['yum'] + ['--enablerepo=' + x for x in repos] + ['clean', 'all'])
def test_03_create_user_proxy(self): core.skip_ok_unless_installed('globus-proxy-utils') self.skip_ok_if(self.__user_proxy_path == '', "User proxy path does not exist.") # OK, software is present, now just check it previous tests did create the proxy already so # we don't do it twice command = ('grid-proxy-info', '-f', self.__user_proxy_path) status, _, _ = core.system(command, True) if int( status ) != 0: # no proxy found for some reason, try to construct a new one command = ('grid-proxy-init', '-out', self.__user_proxy_path) password = core.options.password + '\n' status, _, _ = core.system(command, True, password) self.assert_( status == 0, 'grid-proxy-init for user ' + core.options.username + ' has failed even though globus-proxy-util was present') # we need to have the right permissions on that proxy for glexec to agree to work, # and the easiest way is to copy the file command = ('/bin/cp', self.__user_proxy_path, self.__glexec_client_cert) status, _, _ = core.system(command) os.environ['GLEXEC_CLIENT_CERT'] = self.__glexec_client_cert
def test_04_modify_bestman_conf(self): core.skip_ok_unless_installed('bestman2-server', 'bestman2-client', 'gums-service') bestman_rc_path = '/etc/bestman2/conf/bestman2.rc' old_port = 'securePort=8443' new_port = 'securePort=10443' files.replace(bestman_rc_path, old_port, new_port, backup=False) old_gridmap = 'GridMapFileName=/etc/bestman2/conf/grid-mapfile.empty' new_gridmap = 'GridMapFileName=/etc/grid-security/grid-mapfile' files.replace(bestman_rc_path, old_gridmap, new_gridmap, backup=False) files.replace(bestman_rc_path, 'eventLogLevel=INFO', 'eventLogLevel=DEBUG', backup=False) core.system(('cat', bestman_rc_path)) env_file = '/etc/sysconfig/bestman2' old_auth = 'BESTMAN_GUMS_ENABLED=yes' new_auth = 'BESTMAN_GUMS_ENABLED=no' files.replace(env_file, old_auth, new_auth, backup=False) log4j_path = '/etc/bestman2/properties/log4j.properties' log4j_contents = files.read(log4j_path, as_single_string=True) log4j_contents = log4j_contents.replace('FATAL', 'INFO') files.write(log4j_path, log4j_contents, backup=False)
def test_01_remove_proxy(self): core.skip_ok_unless_installed('myproxy', 'myproxy-server') self.skip_ok_unless(core.state['myproxy.started-server'], 'MyProxy server failed to start') # If there is no pre-existing proxy file, the following command will # produce error output and have exit status 1; because this is the # expected (but not the only valid) case, do not check the output or # exit status. This test exists only to clear out a pre-existing proxy. command = ('myproxy-destroy', '--verbose', '-s', 'localhost', '-l', core.options.username) core.system(command, user=True)
def test_01_remove_proxy(self): core.skip_ok_unless_installed('globus-proxy-utils') # If there is no pre-existing proxy file, the following command will # produce error output and have exit status 1; because this is the # expected (but not the only valid) case, do not check the output or # exit status. This test exists only to clear out a pre-existing proxy. command = ('grid-proxy-destroy', '-debug') core.system(command, user=True)
def dump_log(lines, instance, executable="xrootd"): """Dump the last `lines` lines of an xrootd log file for the given instance.""" # Using tail(1) here because the executable log is actually a nice place # to put the output. core.system([ "tail", "-n", str(lines), f"/var/log/xrootd/{instance}/{executable}.log" ])
def test_01_remove_proxy(self): core.skip_ok_unless_installed('myproxy', 'myproxy-server') self.skip_ok_unless(core.state['myproxy.started-server'], 'MyProxy server failed to start') # If there is no pre-existing proxy file, the following command will # produce error output and have exit status 1; because this is the # expected (but not the only valid) case, do not check the output or # exit status. This test exists only to clear out a pre-existing proxy. command = ('myproxy-destroy', '--verbose', '-s', core.get_hostname(), '-l', core.options.username) core.system(command, user=True)
def test_01_config_mkgridmap(self): core.config['edg.conf'] = '/usr/share/osg-test/edg-mkgridmap.conf' core.skip_ok_unless_installed('edg-mkgridmap', 'voms-server') contents = ('group vomss://%s:8443/voms/%s %s\n' % (socket.getfqdn(), core.config['voms.vo'], core.options.username)) files.write(core.config['edg.conf'], contents, owner='edg') core.system(('cat', core.config['edg.conf']))
def test_02_check_usercert_pass(self): core.skip_ok_unless_installed('globus-proxy-utils') user = pwd.getpwnam(core.options.username) userkey = os.path.join(user.pw_dir, '.globus', 'userkey.pem') command = ('openssl', 'rsa', '-in', userkey, '-passin', 'pass:'******'-text') exit_status, _, _ = core.system(command, user=True) if exit_status == 0: core.system(('grid-proxy-destroy',), user=True) self.fail('user cert has no password')
def test_08_advertise(self): voms.skip_ok_unless_installed() voms.advertise_lsc(core.config['voms.vo'], core.config['certs.hostcert']) files.preserve('/etc/vomses', owner='voms') voms.advertise_vomses(core.config['voms.vo'], core.config['certs.hostcert']) core.system('ls -ldF /etc/*vom*', shell=True) core.system(('find', '/etc/grid-security/vomsdir', '-ls'))
def start_xrootd(instance): svc = "xrootd@%s" % instance if not service.is_running(svc): try: service.check_start(svc, min_up_time=3) except Exception: core.system("tail -n 75 /var/log/xrootd/%s/xrootd.log" % instance, shell=True) raise
def test_08_advertise(self): voms.skip_ok_unless_installed() voms.advertise_lsc(core.config['voms.vo'], core.config['certs.hostcert']) files.preserve('/etc/vomses', owner='voms') voms.advertise_vomses(core.config['voms.vo'], core.config['certs.hostcert']) core.system('ls -ldF /etc/*vom*', shell=True) core.system(('find', '/etc/grid-security/vomsdir', '-ls'))
def test_01_config_mkgridmap(self): core.config['edg.conf'] = '/usr/share/osg-test/edg-mkgridmap.conf' core.skip_ok_unless_installed('edg-mkgridmap', 'voms-admin-server') self.skip_bad_unless(core.state['voms-admin.read-members'], 'Cannot read VO member list') self.skip_bad_unless(core.state['tomcat.started'], 'Tomcat not started') contents = ('group vomss://%s:8443/voms/%s %s\n' % (socket.getfqdn(), core.config['voms.vo'], core.options.username)) files.write(core.config['edg.conf'], contents, owner='edg') core.system(('cat', core.config['edg.conf']))
def test_01_config_mkgridmap(self): core.config['edg.conf'] = '/usr/share/osg-test/edg-mkgridmap.conf' if core.missing_rpm('edg-mkgridmap', 'voms-server'): return contents = ( 'group vomss://%s:8443/voms/%s %s\n' % (socket.getfqdn(), core.config['voms.vo'], core.options.username)) files.write(core.config['edg.conf'], contents, owner='edg') core.system(('cat', core.config['edg.conf']))
def test_02_xrdcp_server_to_local(self): if core.missing_rpm('xrootd-server', 'xrootd-client'): return hostname = socket.getfqdn() temp_source_dir = tempfile.mkdtemp() temp_target_dir = tempfile.mkdtemp() os.chmod(temp_source_dir, 0777) os.chmod(temp_target_dir, 0777) f=open(temp_source_dir+"/copied_file.txt","w") f.write("This is some test data for an xrootd test.") f.close() xrootd_url = 'root://%s/%s/copied_file.txt' % (hostname, temp_source_dir) local_path = temp_target_dir + '/copied_file.txt' command = ('xrdcp', xrootd_url, local_path) status, stdout, stderr = core.system(command, True) fail = core.diagnose('Xrootd xrdcp copy, URL to local', status, stdout, stderr) file_copied = os.path.exists(local_path) shutil.rmtree(temp_source_dir) shutil.rmtree(temp_target_dir) if core.el_release() != 6: self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing') else: self.assertEqual(status, 1, fail) self.assert_(not file_copied, 'Copied file exists')
def test_01_xrdcp_local_to_server(self): core.state['xrootd.copied-to-server'] = False core.skip_ok_unless_installed('xrootd', 'xrootd-client', by_dependency=True) core.skip_ok_unless_installed('globus-proxy-utils') self.skip_bad_unless(core.state['xrootd.started-server'] is True, 'Server not running') temp_dir = tempfile.mkdtemp() core.config['xrootd.tmp-dir'] = temp_dir user = pwd.getpwnam(core.options.username) os.chown(temp_dir, user[2], user[3]) hostname = socket.getfqdn() os.chmod(temp_dir, 0o777) xrootd_url = 'root://%s/%s/copied_file.txt' % (hostname, temp_dir) command = ('xrdcp', '--debug', '3', TestXrootd.__data_path, xrootd_url) status, stdout, stderr = core.system(command, user=True) fail = core.diagnose('xrdcp copy, local to URL', command, status, stdout, stderr) file_copied = os.path.exists(os.path.join(temp_dir, 'copied_file.txt')) if file_copied: core.state['xrootd.copied-to-server'] = True self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing')
def isProbeDataValidInDatabase(self, command, queryFailureString, assertionValue='', atLeastOneRecord=False): """This helper method queries the database for probe related information and based on the passed-in data, determines if the queried information is valid or not.""" status, stdout, _ = core.system(command, shell=True) if status != 0: core.log_message(queryFailureString) return False #Unable to query the database ######If we reached this point, database query was successful. Now, it's time to examine the query output##### if assertionValue != '': result = re.search(assertionValue, stdout, re.IGNORECASE) if result != None: return True #Found the assertionValue in the database else: return False #Unable to find the assertionValue in the database else: #No assertionValue passed in if atLeastOneRecord == True: if int(stdout) < 1: core.log_message("Query did not return one or more records.") return False #Query should return at least one record else: return True #Query returned at least one record else: #"atLeastOneRecord" Flag was not passed in return True
def isProbeDataValidInDatabase(self, command, queryFailureString, assertionValue='', atLeastOneRecord=False): """This helper method queries the database for probe related information and based on the passed-in data, determines if the queried information is valid or not.""" status, stdout, _ = core.system(command, shell=True) if status != 0: core.log_message(queryFailureString) return False #Unable to query the database ######If we reached this point, database query was successful. Now, it's time to examine the query output##### if assertionValue != '': result = re.search(assertionValue, stdout, re.IGNORECASE) if result != None: return True #Found the assertionValue in the database else: return False #Unable to find the assertionValue in the database else: #No assertionValue passed in if atLeastOneRecord == True: if int(stdout) < 1: core.log_message( "Query did not return one or more records.") return False #Query should return at least one record else: return True #Query returned at least one record else: #"atLeastOneRecord" Flag was not passed in return True
def test_04_copy_server_to_local_uberftp_parallel(self): core.skip_ok_unless_installed(*self.required_rpms) self.skip_bad_unless(core.state['gridftp.running-server'] is True, 'GridFTP not running') hostname = socket.getfqdn() temp_dir_source = tempfile.mkdtemp() temp_dir_dest = tempfile.mkdtemp() os.chmod(temp_dir_source, 0o777) os.chmod(temp_dir_dest, 0o777) filename = 'testfile_10MB' full_path = (os.path.join(temp_dir_source, filename)) command = ('dd', 'if=/dev/zero', 'of=' + full_path, 'bs=10485760', 'count=1') core.check_system(command, 'Create test file with dd', user=True) ftp_cmd = ('cd %s; lcd %s; get %s' % (temp_dir_source, temp_dir_dest, filename)) command = ('uberftp', '-parallel', '10', hostname, ftp_cmd) status, stdout, stderr = core.system(command, True) fail = core.diagnose('UberFTP copy, local to URL', command, status, stdout, stderr) file_copied = os.path.exists(os.path.join(temp_dir_dest, filename)) shutil.rmtree(temp_dir_source) shutil.rmtree(temp_dir_dest) self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing')
def test_14_execute_dcache_storage(self): # Malformed XML errors due to network issues (SOFTWARE-1748) core.state['gratia.dcache-whitelisted-error'] = False whitelisted_errors = ['The element type "metric" must be terminated by the matching end-tag "</metric>".', 'XML document structures must start and end within the same entity.'] core.skip_ok_unless_installed('gratia-probe-dcache-storage', 'gratia-service') core.state['gratia.dcache-storage-running'] = False self.skip_bad_if(core.state['gratia.dcache-logs-copied'] == False) if os.path.exists(core.config['gratia.log.file']): core.state['gratia.log.stat'] = os.stat(core.config['gratia.log.file']) core.log_message('stat.st_ino is: ' + str(core.state['gratia.log.stat'].st_ino)) core.log_message('stat.st_size is: ' + str(core.state['gratia.log.stat'].st_size)) command = ('/usr/share/gratia/dCache-storage/dCache-storage_meter.cron.sh',) status, stdout, stderr = core.system(command) if status != 0: for error in whitelisted_errors: if error in stdout: core.state['gratia.dcache-whitelisted-error'] = True break if not core.state['gratia.dcache-whitelisted-error']: self.fail(core.diagnose('Unable to execute dCache-storage.', command, status, stdout, stderr)) core.config['gratia.dcache-temp-dir'] = core.config['gratia.tmpdir.prefix'] + "subdir.dCache-storage" + \ core.config['gratia.tmpdir.postfix'] if core.state['gratia.database-installed'] == True: result = self.isProbeOutboxDirEmpty(core.config['gratia.dcache-temp-dir']) self.assert_(result, 'dCache-storage outbox check failed.') core.state['gratia.dcache-storage-running'] = True
def test_01_xrdcp_local_to_server(self): core.skip_ok_unless_installed('xrootd', 'xrootd-client', by_dependency=True) if core.config['xrootd.gsi'] == "ON": core.skip_ok_unless_installed('globus-proxy-utils') self.skip_bad_unless(core.state['xrootd.started-server'] is True, 'Server not running') temp_dir = "/tmp/vdttest" hostname = socket.getfqdn() if core.config['xrootd.gsi'] == "ON": if not os.path.exists(temp_dir): os.mkdir(temp_dir) user = pwd.getpwnam(core.options.username) os.chown(temp_dir, user[2], user[3]) else: temp_dir = tempfile.mkdtemp() os.chmod(temp_dir, 0o777) xrootd_url = 'root://%s:%d/%s/copied_file.txt' % (hostname, core.config['xrootd.port'], temp_dir) command = ('xrdcp', '--debug', '3', TestXrootd.__data_path, xrootd_url) status, stdout, stderr = core.system(command, user=True) fail = core.diagnose('xrdcp copy, local to URL', command, status, stdout, stderr) file_copied = os.path.exists(os.path.join(temp_dir, 'copied_file.txt')) if core.config['xrootd.multiuser'] != "ON": shutil.rmtree(temp_dir) self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing')
def test_07_remove_test_user(self): if not core.state['general.user_added']: core.log_message('Did not add user') return username = core.options.username password_entry = pwd.getpwnam(username) globus_dir = os.path.join(password_entry.pw_dir, '.globus') # Remove certs in case userdel fails if core.state['general.user_cert_created']: files.remove(os.path.join(globus_dir, 'usercert.pem')) files.remove(os.path.join(globus_dir, 'userkey.pem')) # Get list of PIDs owned by the test user command = ('ps', '-U', username, '-u', username, '-o', 'pid=') _, output, _ = core.system(command) # Take no prisoners for pid in output.splitlines(): try: os.kill(int(pid), signal.SIGKILL) except OSError: continue command = ('userdel', username) core.check_system(command, "Remove user '%s'" % (username)) files.remove(os.path.join('/var/spool/mail', username)) shutil.rmtree(password_entry.pw_dir)
def test_03_xrdcp_server_to_local(self): core.skip_ok_unless_installed('xrootd', 'xrootd-client', by_dependency=True) core.skip_ok_unless_installed('globus-proxy-utils') self.skip_bad_unless(core.state['xrootd.started-server'] is True, 'Server not running') hostname = socket.getfqdn() temp_source_dir = tempfile.mkdtemp() temp_target_dir = tempfile.mkdtemp() os.chmod(temp_source_dir, 0o777) os.chmod(temp_target_dir, 0o777) f = open(temp_source_dir + "/copied_file.txt", "w") f.write("This is some test data for an xrootd test.") f.close() xrootd_url = 'root://%s/%s/copied_file.txt' % (hostname, temp_source_dir) local_path = temp_target_dir + '/copied_file.txt' command = ('xrdcp', '--debug', '3', xrootd_url, local_path) status, stdout, stderr = core.system(command, user=True) fail = core.diagnose('Xrootd xrdcp copy, URL to local', command, status, stdout, stderr) file_copied = os.path.exists(local_path) shutil.rmtree(temp_source_dir) shutil.rmtree(temp_target_dir) self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing')
def test_04_copy_server_to_local_uberftp_parallel(self): core.skip_ok_unless_installed(*self.required_rpms) self.skip_bad_unless(core.state['gridftp.running-server'] is True, 'GridFTP not running') hostname = socket.getfqdn() temp_dir_source = tempfile.mkdtemp() temp_dir_dest = tempfile.mkdtemp() os.chmod(temp_dir_source, 0o777) os.chmod(temp_dir_dest, 0o777) filename = 'testfile_10MB' full_path = (os.path.join(temp_dir_source, filename)) command = ('dd', 'if=/dev/zero', 'of=' + full_path, 'bs=10485760', 'count=1') core.check_system(command, 'Create test file with dd', user=True) ftp_cmd = ('cd %s; lcd %s; get %s' % (temp_dir_source, temp_dir_dest, filename)) command = ('uberftp', '-parallel','10', hostname, ftp_cmd) status, stdout, stderr = core.system(command, True) fail = core.diagnose('UberFTP copy, local to URL', command, status, stdout, stderr) file_copied = os.path.exists(os.path.join(temp_dir_dest, filename)) shutil.rmtree(temp_dir_source) shutil.rmtree(temp_dir_dest) self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing')
def test_07_remove_test_user(self): if not core.state['general.user_added']: core.log_message('Did not add user') return username = core.options.username password_entry = pwd.getpwnam(username) globus_dir = os.path.join(password_entry.pw_dir, '.globus') # Remove certs in case userdel fails if core.state['general.user_cert_created']: files.remove(os.path.join(globus_dir, 'usercert.pem')) files.remove(os.path.join(globus_dir, 'userkey.pem')) # Get list of PIDs owned by the test user command = ('ps', '-U', username, '-u', username, '-o', 'pid=') _, output, _ = core.system(command) # Take no prisoners for pid in output.splitlines(): try: os.kill(int(pid), signal.SIGKILL) except OSError: continue command = ('userdel', username) core.check_system(command, "Remove user '%s'" % (username)) files.remove(os.path.join('/var/spool/mail', username)) shutil.rmtree(password_entry.pw_dir)
def test_04_glexec_switch_id(self): core.skip_ok_unless_installed('glexec', 'globus-proxy-utils') command = ('grid-proxy-info', '-f', self.__user_proxy_path) status, stdout, _ = core.system(command, True) if int(status) != 0: # no proxy found even after previous checks, have to skip self.skip_bad('suitable proxy not found') return command = ('/usr/sbin/glexec', '/usr/bin/id', '-u') status, stdout, _ = core.system(command) switched_id = stdout.rstrip() self.assert_(self.__uid == switched_id, 'Glexec identity switch from root to user ' + core.options.username + ' failed')
def dbdump(destfile, database=None): """Dump the contents of one or all databases to the given file `destfile` must be a path the user can write to. If `database` is specified, only the given database is dumped; otherwise, all databases are dumped. The output is suitable for feeding back into `mysql` as a script. """ command = "mysqldump --skip-comments --skip-extended-insert -u root " if database: command += re.escape(database) else: command += "--all-databases" command += ">" + re.escape(destfile) core.system(command, user=None, stdin=None, log_output=False, shell=True)
def test_01_xrdcp_local_to_server(self): if core.missing_rpm('xrootd-server', 'xrootd-client'): return hostname = socket.getfqdn() if core.config['xrootd.gsi'] == "ON": temp_dir="/tmp/vdttest" if not os.path.exists(temp_dir): os.mkdir(temp_dir) else: temp_dir = tempfile.mkdtemp() os.chmod(temp_dir, 0777) xrootd_url = 'root://%s/%s/copied_file.txt' % (hostname, temp_dir) command = ('xrdcp', TestXrootd.__data_path , xrootd_url) status, stdout, stderr = core.system(command, True) fail = core.diagnose('xrdcp copy, local to URL', status, stdout, stderr) file_copied = os.path.exists(os.path.join(temp_dir, 'copied_file.txt')) shutil.rmtree(temp_dir) if core.el_release() != 6: self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing') else: self.assertEqual(status, 1, fail) self.assert_(not file_copied, 'Copied file existed somehow')
def test_024_rsv_control_bad_arg(self): core.skip_ok_unless_installed('rsv') command = ('rsv-control', '--kablooey') ret, _, _ = core.system(command, 'rsv-control --kablooey') self.assert_(ret != 0) return
def test_03_xrdcp_server_to_local(self): core.skip_ok_unless_installed('xrootd', 'xrootd-client', by_dependency=True) if core.config['xrootd.gsi'] == "ON": core.skip_ok_unless_installed('globus-proxy-utils') self.skip_bad_unless(core.state['xrootd.started-server'] is True, 'Server not running') hostname = socket.getfqdn() temp_source_dir = tempfile.mkdtemp() temp_target_dir = tempfile.mkdtemp() os.chmod(temp_source_dir, 0o777) os.chmod(temp_target_dir, 0o777) f = open(temp_source_dir + "/copied_file.txt", "w") f.write("This is some test data for an xrootd test.") f.close() xrootd_url = 'root://%s:%d/%s/copied_file.txt' % (hostname, core.config['xrootd.port'], temp_source_dir) local_path = temp_target_dir + '/copied_file.txt' command = ('xrdcp', '--debug', '3', xrootd_url, local_path) status, stdout, stderr = core.system(command, user=True) fail = core.diagnose('Xrootd xrdcp copy, URL to local', command, status, stdout, stderr) file_copied = os.path.exists(local_path) shutil.rmtree(temp_source_dir) shutil.rmtree(temp_target_dir) self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing')
def test_04_copy_server_to_local_uberftp_parallel(self): if core.missing_rpm('globus-gridftp-server-progs', 'globus-ftp-client', 'globus-proxy-utils', 'globus-gass-copy-progs', 'uberftp'): return hostname = socket.getfqdn() temp_dir_source = tempfile.mkdtemp() temp_dir_dest = tempfile.mkdtemp() os.chmod(temp_dir_source, 0777) os.chmod(temp_dir_dest, 0777) filename = 'testfile_10MB' full_path = (os.path.join(temp_dir_source, filename)) command = ('dd', 'if=/dev/zero', 'of=' + full_path, 'bs=10485760', 'count=1') core.check_system(command, 'Create test file with dd', user=True) ftp_cmd = ('cd %s; lcd %s; get %s' % (temp_dir_source, temp_dir_dest, filename)) command = ('uberftp', '-parallel', '10', hostname, ftp_cmd) status, stdout, stderr = core.system(command, True) fail = core.diagnose('UberFTP copy, local to URL', status, stdout, stderr) file_copied = os.path.exists(os.path.join(temp_dir_dest, filename)) shutil.rmtree(temp_dir_source) shutil.rmtree(temp_dir_dest) self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing')
def test_04_remove_server_file(self): command = ('srm-rm', self.get_srm_url(), '-debug') status, stdout, stderr = core.system(command, True) fail = core.diagnose('Bestman remove, URL file', command, status, stdout, stderr) file_removed = not os.path.exists(TestBestman.__remote_path) self.assertEqual(status, 0, fail) self.assert_(file_removed, 'Copied file still exists') files.remove(TestBestman.__temp_dir)
def test_04_remove_server_file(self): command = ('srm-rm', self.get_srm_url(), '-debug') status, stdout, stderr = core.system(command, True) fail = core.diagnose('Bestman remove, URL file', command, status, stdout, stderr) file_removed = not os.path.exists(TestBestman.__remote_path) self.assertEqual(status, 0, fail) self.assert_(file_removed, 'Copied file still exists') files.remove(TestBestman.__temp_dir)
def test_03_copy_server_to_local(self): command = ('srm-copy', self.get_srm_url(), 'file://' + TestBestman.__local_path, '-debug') status, stdout, stderr = core.system(command, True) fail = core.diagnose('Bestman copy, URL to local', command, status, stdout, stderr) file_copied = os.path.exists(TestBestman.__local_path) self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing') files.remove(TestBestman.__local_path)
def test_03_remove_server_file_lcg_util(self): command = ('lcg-del', '-v', '-b', '-l', '-D', 'srmv2', self.get_srm_url()) status, stdout, stderr = core.system(command, True) fail = core.diagnose('lcg-util remove, URL file', command, status, stdout, stderr) file_removed = not os.path.exists(TestLCGUtil.__remote_path) self.assertEqual(status, 0, fail) self.assert_(file_removed, 'Copied file still exists') files.remove(TestLCGUtil.__temp_dir)
def test_02_copy_server_to_local_lcg_util(self): command = ('lcg-cp', '-v', '-b', '-D', 'srmv2', self.get_srm_url(), 'file://' + TestLCGUtil.__local_path) status, stdout, stderr = core.system(command, True) fail = core.diagnose('lcg-util copy, URL to local', command, status, stdout, stderr) file_copied = os.path.exists(TestLCGUtil.__local_path) self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing') files.remove(TestLCGUtil.__local_path)
def test_03_proxypath(self): # Grab the path of the proxy created for the proxy test core.skip_ok_unless_installed('globus-proxy-utils', 'myproxy', 'myproxy-server') self.skip_ok_unless(core.state['myproxy.started-server'], 'MyProxy server failed to start') command = ('grid-proxy-info', '-path') _, proxypath, _ = core.system(command, user=True) core.state['proxy.path'] = proxypath.split('\n')[0]
def test_01_copy_local_to_server_lcg_util(self): self.setup_temp_paths() os.chmod(TestLCGUtil.__temp_dir, 0777) command = ('lcg-cp', '-v', '-b', '-D', 'srmv2', 'file://' + TestLCGUtil.__data_path, self.get_srm_url()) status, stdout, stderr = core.system(command, True) fail = core.diagnose('lcg-util copy, local to URL', command, status, stdout, stderr) file_copied = os.path.exists(TestLCGUtil.__remote_path) self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing')
def test_04_bad_voms_proxy_init(self): core.skip_ok_unless_installed('voms-admin-server', 'voms-admin-client', 'voms-clients', by_dependency=True) self.skip_bad_unless(core.state['voms.started-webapp']) command = ('voms-proxy-init', '-voms', core.config['voms.vo'] + ':/Bogus') password = core.options.password + '\n' status, stdout, _ = core.system(command, True, password) self.assertNotEqual(status, 0, 'voms-proxy-init fails on bad group') self.assert_('Unable to satisfy' in stdout, 'voms-proxy-init failure message')
def test_02_copy_local_to_server(self): self.setup_temp_paths() os.chmod(TestBestman.__temp_dir, 0777) command = ('srm-copy', 'file://' + TestBestman.__data_path, self.get_srm_url(), '-debug') status, stdout, stderr = core.system(command, True) fail = core.diagnose('Bestman copy, local to URL', command, status, stdout, stderr) file_copied = os.path.exists(TestBestman.__remote_path) self.assertEqual(status, 0, fail) self.assert_(file_copied, 'Copied file missing')
def test_04_bad_voms_proxy_init(self): voms.skip_ok_unless_installed() self.skip_bad_unless(core.state['voms.added-user']) command = ('voms-proxy-init', '-voms', core.config['voms.vo'] + ':/Bogus') password = core.options.password + '\n' status, stdout, _ = core.system(command, True, password) self.assertNotEqual(status, 0, 'voms-proxy-init fails on bad group') self.assert_('Unable to satisfy' in stdout, 'voms-proxy-init failure message')
def test_01_check_gridmap(self): core.skip_ok_unless_installed('glexec') key_dn = '"'+ core.config['user.cert_subject'] + '"' + ' ' + core.options.username command = ('/bin/grep', key_dn, self.__grid_mapfile) status, _, _ = core.system(command) self.assert_(status == 0, 'Grid-mapfile entry for user '+core.options.username+' missing') TestGlexec.__good_gridmap = True
def test_03_remove_server_file_gfal2_util(self): core.skip_ok_unless_installed('bestman2-server', 'gfal2-plugin-srm', 'gums-service') self.skip_bad_unless(core.state['bestman.server-running'], 'bestman server not running') command = ('gfal-rm', '-v', self.get_srm_url()) status, stdout, stderr = core.system(command, True) fail = core.diagnose('gfal2-util remove, URL file', command, status, stdout, stderr) file_removed = not os.path.exists(TestGFAL2Util.__remote_path) self.assertEqual(status, 0, fail) self.assert_(file_removed, 'Copied file still exists') files.remove(TestGFAL2Util.__temp_dir)
def test_08_backups(self): record_is_clear = True if len(files._backups) > 0: details = '' for backup_id, backup_path in files._backups.items(): details += "-- Backup of '%s' for '%s' in '%s'\n" % (backup_id[0], backup_id[1], backup_path) core.log_message('Backups remain in backup dictionary:\n' + details) record_is_clear = False actual_is_clear = True if os.path.isdir(files._backup_directory): backups = os.listdir(files._backup_directory) if len(backups) > 0: core.log_message("Files remain in '%s:'" % (files._backup_directory)) core.system('ls -lF ' + files._backup_directory, shell=True) actual_is_clear = False shutil.rmtree(files._backup_directory, ignore_errors=True) self.assert_(record_is_clear and actual_is_clear, 'Backups were not restored fully')