def test_10_install_vo_webapp(self):
core.state['voms.installed-vo-webapp'] = False
core.skip_ok_unless_installed('voms-admin-server')
self.skip_ok_if(os.path.exists(core.config['voms.vo-webapp']), 'apparently installed')
service.check_start('voms-admin')
core.state['voms.installed-vo-webapp'] = True
def test_02_start_slurmdbd(self):
core.state['slurmdbd.started-service'] = False
self.slurm_reqs()
core.skip_ok_unless_installed('slurm-slurmdbd')
self.skip_bad_unless(mysql.is_running(), 'slurmdbd requires mysql')
core.config['slurmdbd.config'] = '/etc/slurm/slurmdbd.conf'
core.config['slurmdbd.user'] = "******"
core.config['slurmdbd.name'] = "osg_test_slurmdb"
mysql.check_execute("create database %s; " % core.config['slurmdbd.name'], 'create slurmdb')
mysql.check_execute("create user %s; " % core.config['slurmdbd.user'], 'add slurmdb user')
mysql.check_execute("grant usage on *.* to %s; " % core.config['slurmdbd.user'], 'slurmdb user access')
mysql.check_execute("grant all privileges on %s.* to %s identified by '%s'; " % (core.config['slurmdbd.name'],
core.config['slurmdbd.user'],
core.options.password),
'slurmdb user permissions')
mysql.check_execute("flush privileges;", 'reload privileges')
db_config_vals = {'name':core.config['slurmdbd.name'],
'user':core.config['slurmdbd.user'].split('\'')[1],
'pass':core.options.password}
files.write(core.config['slurmdbd.config'],
SLURMDBD_CONFIG % db_config_vals,
owner='slurm',
chmod=0o644)
service.check_start('slurmdbd')
# Adding the cluster to the database
command = ('sacctmgr', '-i', 'add', 'cluster', CLUSTER_NAME)
core.check_system(command, 'add slurm cluster')
def test_03_start_xrootd(self):
core.config['xrootd_tpc_service_1'] = "xrootd@third-party-copy-1"
core.config['xrootd_tpc_service_2'] = "xrootd@third-party-copy-2"
service.check_start(core.config['xrootd_tpc_service_1'], log_to_check = '/var/log/xrootd/third-party-copy-1/xrootd.log')
service.check_start(core.config['xrootd_tpc_service_2'], log_to_check = '/var/log/xrootd/third-party-copy-2/xrootd.log')
core.state['xrootd.started-http-server-1'] = True
core.state['xrootd.started-http-server-2'] = True
def test_05_start_tomcat(self):
core.skip_ok_unless_installed(tomcat.pkgname())
core.state['tomcat.started'] = False
catalina_log = tomcat.catalinafile()
initial_stat = core.get_stat(catalina_log)
tomcat_sentinel = r'Server startup in \d+ ms'
# Bump log level
core.config['tomcat.logging-conf'] = os.path.join(tomcat.sysconfdir(), 'logging.properties')
files.append(core.config['tomcat.logging-conf'], 'org.apache.catalina.level = %s\n' % 'FINER',
owner='tomcat', backup=True)
old_str = "1catalina.org.apache.juli.FileHandler.prefix = catalina."
repl_str = ("1catalina.org.apache.juli.FileHandler.prefix = catalina\n"
"1catalina.org.apache.juli.FileHandler.rotatable = false")
files.replace(core.config['tomcat.logging-conf'], old_str, repl_str,
owner='tomcat', backup=False)
service.check_start(tomcat.pkgname())
if core.options.nightly:
timeout = 3600.0
else:
timeout = 1200.0
line, gap = core.monitor_file(catalina_log, initial_stat, tomcat_sentinel, timeout)
self.assert_(line is not None, 'Tomcat did not start within the %d min window' % int(timeout/60))
core.state['tomcat.started'] = True
core.log_message('Tomcat started after %.1f seconds' % gap)
def test_05_start_pbs(self):
core.state['pbs_server.started-service'] = False
core.state['torque.nodes-up'] = False
core.skip_ok_unless_installed(*self.required_rpms, by_dependency=True)
self.skip_bad_unless(service.is_running('trqauthd'), 'pbs_server requires trqauthd')
self.skip_ok_if(service.is_running('pbs_server'), 'pbs server already running')
server_log = '/var/log/torque/server_logs/' + date.today().strftime('%Y%m%d')
try:
server_log_stat = os.stat(server_log)
except OSError:
server_log_stat = None
service.check_start('pbs_server')
# Wait until the server is up before writing the rest of the config
core.monitor_file(server_log, server_log_stat, '.*Server Ready.*', 60.0)
core.check_system("echo '%s' | qmgr %s" % (self.pbs_config, core.get_hostname()),
"Configuring pbs server",
shell=True)
# wait up to 5 minutes for the server to recognize the node
start_time = time.time()
while (time.time() - start_time) < 600:
command = ('/usr/bin/qnodes', '-s', core.get_hostname())
stdout, _, fail = core.check_system(command, 'Get pbs node info')
self.assert_(stdout.find('error') == -1, fail)
if stdout.find('state = free'):
core.state['torque.nodes-up'] = True
break
if not core.state['torque.nodes-up']:
self.fail('PBS nodes not coming up')
def test_04_start_condorce(self):
if core.el_release() >= 7:
core.config[
'condor-ce.lockfile'] = '/var/lock/condor-ce/htcondor-ceLock'
else:
core.config['condor-ce.lockfile'] = '/var/lock/subsys/condor-ce'
core.state['condor-ce.started-service'] = False
core.state['condor-ce.schedd-ready'] = False
core.skip_ok_unless_installed('condor', 'htcondor-ce',
'htcondor-ce-client')
core.config['condor-ce.collectorlog'] = condor.ce_config_val(
'COLLECTOR_LOG')
if service.is_running('condor-ce'):
core.state['condor-ce.schedd-ready'] = True
self.skip_ok('already running')
stat = core.get_stat(core.config['condor-ce.collectorlog'])
service.check_start('condor-ce', timeout=20)
if condor.wait_for_daemon(core.config['condor-ce.collectorlog'], stat,
'Schedd', 300.0):
core.state['condor-ce.schedd-ready'] = True
def test_03_start_slurm(self):
core.config['slurm.service-name'] = 'slurmd'
core.config['slurm.ctld-service-name'] = 'slurmctld'
core.state['%s.started-service' %
core.config['slurm.service-name']] = False
self.slurm_reqs()
self.skip_ok_if(service.is_running(core.config['slurm.service-name']),
'slurm already running')
stat = core.get_stat(CTLD_LOG)
command = ['slurmctld']
core.check_system(command, 'enable slurmctld')
service.check_start(core.config['slurm.service-name'])
service.check_start(core.config['slurm.ctld-service-name'])
core.monitor_file(
CTLD_LOG, stat,
'slurm_rpc_node_registration complete for %s' % SHORT_HOSTNAME,
60.0)
log_stat = core.get_stat(SLURM_LOG)
core.monitor_file(SLURM_LOG, log_stat, 'slurmd started', 60.0)
command = [
'scontrol', 'update',
'nodename=%s' % SHORT_HOSTNAME, 'state=idle'
]
core.check_system(command, 'enable slurm node')
def test_03_start_xrootd(self):
core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', by_dependency=True)
core.config['xrootd_tpc_service_1'] = "xrootd@third-party-copy-1"
core.config['xrootd_tpc_service_2'] = "xrootd@third-party-copy-2"
service.check_start(core.config['xrootd_tpc_service_1'], log_to_check = '/var/log/xrootd/third-party-copy-1/xrootd.log')
service.check_start(core.config['xrootd_tpc_service_2'], log_to_check = '/var/log/xrootd/third-party-copy-2/xrootd.log')
core.state['xrootd.started-http-server-1'] = True
core.state['xrootd.started-http-server-2'] = True
def test_04_start_myproxy(self):
core.state['myproxy.started-server'] = False
core.skip_ok_unless_installed('myproxy-server')
self.skip_ok_if(os.path.exists(core.config['myproxy.lock-file']), 'apparently running')
service.check_start('myproxy-server')
core.state['myproxy.started-server'] = True
def test_03_start_xrootd(self):
core.skip_ok_unless_installed('xrootd', by_dependency=True)
if core.el_release() < 7:
core.config['xrootd_service'] = "xrootd"
else:
core.config['xrootd_service'] = "xrootd@clustered"
service.check_start(core.config['xrootd_service'])
core.state['xrootd.started-server'] = True
def start_xrootd(instance):
svc = "xrootd@%s" % instance
if not service.is_running(svc):
try:
service.check_start(svc, min_up_time=3)
except Exception:
core.system("tail -n 75 /var/log/xrootd/%s/xrootd.log" % instance,
shell=True)
raise
def test_03_start_xrootd(self):
if core.rpm_is_installed("xrootd-multiuser"):
core.config['xrootd_tpc_service_1'] = "xrootd-privileged@third-party-copy-1"
core.config['xrootd_tpc_service_2'] = "xrootd-privileged@third-party-copy-2"
else:
core.config['xrootd_tpc_service_1'] = "xrootd@third-party-copy-1"
core.config['xrootd_tpc_service_2'] = "xrootd@third-party-copy-2"
service.check_start(core.config['xrootd_tpc_service_1'], min_up_time = 5)
service.check_start(core.config['xrootd_tpc_service_2'], min_up_time = 5)
core.state['xrootd.started-http-server-1'] = True
core.state['xrootd.started-http-server-2'] = True
def test_04_start_xrootd(self):
core.skip_ok_unless_installed('xrootd', by_dependency=True)
if core.el_release() < 7:
core.config['xrootd_service'] = "xrootd"
elif core.config['xrootd.multiuser']:
core.config['xrootd_service'] = "xrootd-privileged@clustered"
else:
core.config['xrootd_service'] = "xrootd@clustered"
service.check_start(core.config['xrootd_service'])
core.state['xrootd.started-server'] = True
def test_04_start_xrootd(self):
core.skip_ok_unless_installed('xrootd', 'globus-proxy-utils', by_dependency=True)
if core.el_release() < 7:
core.config['xrootd_service'] = "xrootd"
elif core.config['xrootd.multiuser']:
core.config['xrootd_service'] = "xrootd-privileged@standalone"
else:
core.config['xrootd_service'] = "xrootd@standalone"
service.check_start(core.config['xrootd_service'])
core.state['xrootd.started-server'] = True
def test_01_start_munge(self):
core.config['munge.keyfile'] = '/etc/munge/munge.key'
core.state['munge.started-service'] = False
core.skip_ok_unless_installed('munge')
self.skip_ok_if(service.is_running('munge'), 'already running')
files.preserve(core.config['munge.keyfile'], 'munge')
command = ('/usr/sbin/create-munge-key', '-f',)
stdout, _, fail = core.check_system(command, 'Create munge key')
self.assert_(stdout.find('error') == -1, fail)
service.check_start('munge')
def test_03_start_trqauthd(self):
core.state['trqauthd.started-service'] = False
core.config['torque.pbs-servername-file'] = '/var/lib/torque/server_name'
core.skip_ok_unless_installed(*self.required_rpms, by_dependency=True)
self.skip_ok_if(service.is_running('trqauthd'), 'trqauthd is already running')
# set hostname as servername instead of localhost
# config required before starting trqauthd
files.write(core.config['torque.pbs-servername-file'],
"%s" % core.get_hostname(),
owner='pbs')
service.check_start('trqauthd')
def test_01_start_gridftp(self):
core.state['gridftp.started-server'] = False
core.state['gridftp.running-server'] = False
core.skip_ok_unless_installed('globus-gridftp-server-progs')
if service.is_running('globus-gridftp-server'):
core.state['gridftp.running-server'] = True
return
service.check_start('globus-gridftp-server')
core.state['gridftp.running-server'] = True
core.state['gridftp.started-server'] = True
def test_01_start_gridftp(self):
core.state['gridftp.started-server'] = False
core.state['gridftp.running-server'] = False
core.skip_ok_unless_installed('globus-gridftp-server-progs')
if service.is_running('globus-gridftp-server'):
core.state['gridftp.running-server'] = True
return
service.check_start('globus-gridftp-server')
core.state['gridftp.running-server'] = True
core.state['gridftp.started-server'] = True
def test_01_start_condor_cron(self):
core.state['condor-cron.started-service'] = False
core.state['condor-cron.running-service'] = False
core.skip_ok_unless_installed('condor-cron')
if service.is_running('condor-cron', timeout=1):
core.state['condor-cron.running-service'] = True
self.skip_ok('already running')
service.check_start('condor-cron')
core.state['condor-cron.started-service'] = True
core.state['condor-cron.running-service'] = True
def test_03_start_trqauthd(self):
core.state['trqauthd.started-service'] = False
core.config[
'torque.pbs-servername-file'] = '/var/lib/torque/server_name'
core.skip_ok_unless_installed(*self.required_rpms, by_dependency=True)
self.skip_ok_if(service.is_running('trqauthd'),
'trqauthd is already running')
# set hostname as servername instead of localhost
# config required before starting trqauthd
files.write(core.config['torque.pbs-servername-file'],
"%s" % core.get_hostname(),
owner='pbs')
service.check_start('trqauthd')
def test_01_start_condor_cron(self):
core.state['condor-cron.started-service'] = False
core.state['condor-cron.running-service'] = False
core.skip_ok_unless_installed('condor-cron')
if service.is_running('condor-cron', timeout=1):
core.state['condor-cron.running-service'] = True
self.skip_ok('already running')
service.check_start('condor-cron')
core.state['condor-cron.started-service'] = True
core.state['condor-cron.running-service'] = True
def test_01_start_mom(self):
core.state['pbs_mom.started-service'] = False
core.skip_ok_unless_installed(*self.required_rpms, by_dependency=True)
self.skip_ok_if(service.is_running('pbs_mom'), 'PBS mom already running')
core.config['torque.mom-config'] = '/var/lib/torque/mom_priv/config'
files.write(core.config['torque.mom-config'],
"$pbsserver %s\n" % core.get_hostname(),
owner='pbs')
core.config['torque.mom-layout'] = '/var/lib/torque/mom_priv/mom.layout'
files.write(core.config['torque.mom-layout'],
"nodes=0",
owner='pbs')
service.check_start('pbs_mom')
def test_09_start_voms(self):
core.state['voms.started-server'] = False
voms.skip_ok_unless_installed()
self.skip_ok_if(os.path.exists(core.config['voms.lock-file']), 'apparently running')
if core.el_release() < 7:
core.config['voms_service'] = 'voms'
else:
core.config['voms_service'] = 'voms@' + core.config['voms.vo']
service.check_start(core.config['voms_service'])
core.state['voms.started-server'] = True
def test_09_start_voms(self):
core.state['voms.started-server'] = False
voms.skip_ok_unless_installed()
self.skip_ok_if(os.path.exists(core.config['voms.lock-file']), 'apparently running')
if core.el_release() < 7:
core.config['voms_service'] = 'voms'
else:
core.config['voms_service'] = 'voms@' + core.config['voms.vo']
service.check_start(core.config['voms_service'])
core.state['voms.started-server'] = True
def test_01_start_munge(self):
core.config['munge.keyfile'] = '/etc/munge/munge.key'
core.state['munge.started-service'] = False
core.skip_ok_unless_installed('munge')
self.skip_ok_if(service.is_running('munge'), 'already running')
files.preserve(core.config['munge.keyfile'], 'munge')
command = (
'/usr/sbin/create-munge-key',
'-f',
)
stdout, _, fail = core.check_system(command, 'Create munge key')
self.assert_(stdout.find('error') == -1, fail)
service.check_start('munge')
def test_01_start_mom(self):
core.state['pbs_mom.started-service'] = False
core.skip_ok_unless_installed(*self.required_rpms, by_dependency=True)
self.skip_ok_if(service.is_running('pbs_mom'),
'PBS mom already running')
core.config['torque.mom-config'] = '/var/lib/torque/mom_priv/config'
files.write(core.config['torque.mom-config'],
"$pbsserver %s\n" % core.get_hostname(),
owner='pbs')
core.config[
'torque.mom-layout'] = '/var/lib/torque/mom_priv/mom.layout'
files.write(core.config['torque.mom-layout'], "nodes=0", owner='pbs')
service.check_start('pbs_mom')
def test_01_start_condor(self):
core.state['condor.running-service'] = False
core.skip_ok_unless_installed('condor')
core.config['condor.collectorlog'] = condor.config_val('COLLECTOR_LOG')
if service.is_running('condor'):
core.state['condor.running-service'] = True
return
core.config['condor.collectorlog_stat'] = core.get_stat(
core.config['condor.collectorlog'])
service.check_start('condor')
core.state['condor.started-service'] = True
core.state['condor.running-service'] = True
def test_02_start_rsv(self):
core.config['rsv.lockfile'] = "/var/lock/subsys/rsv"
core.state['rsv.started-service'] = False
core.state['rsv.running-service'] = False
core.skip_ok_unless_installed('rsv')
# Is RSV already running?
if os.path.exists(core.config['rsv.lockfile']):
core.state['rsv.running-service'] = True
self.skip_ok('already running')
# Before we start RSV, make sure Condor-Cron is up
self.skip_bad_unless(core.state['condor-cron.running-service'], 'Condor-Cron not running')
service.check_start('rsv')
core.state['rsv.started-service'] = True
core.state['rsv.running-service'] = True
def test_03_start_xrootd_tpc(self):
if core.rpm_is_installed("xrootd-multiuser"):
core.config[
'xrootd_tpc_service_1'] = "xrootd-privileged@third-party-copy-1"
core.config[
'xrootd_tpc_service_2'] = "xrootd-privileged@third-party-copy-2"
else:
core.config['xrootd_tpc_service_1'] = "xrootd@third-party-copy-1"
core.config['xrootd_tpc_service_2'] = "xrootd@third-party-copy-2"
if core.options.manualrun:
files.preserve_and_remove(xrootd.logfile("third-party-copy-1"),
owner="xrootd")
files.preserve_and_remove(xrootd.logfile("third-party-copy-2"),
owner="xrootd")
service.check_start(core.config['xrootd_tpc_service_1'], min_up_time=5)
service.check_start(core.config['xrootd_tpc_service_2'], min_up_time=5)
core.state['xrootd.started-http-server-1'] = True
core.state['xrootd.started-http-server-2'] = True
def test_02_start_rsv(self):
core.config['rsv.lockfile'] = "/var/lock/subsys/rsv"
core.state['rsv.started-service'] = False
core.state['rsv.running-service'] = False
core.skip_ok_unless_installed('rsv')
# Is RSV already running?
if os.path.exists(core.config['rsv.lockfile']):
core.state['rsv.running-service'] = True
self.skip_ok('already running')
# Before we start RSV, make sure Condor-Cron is up
self.skip_bad_unless(core.state['condor-cron.running-service'],
'Condor-Cron not running')
service.check_start('rsv')
core.state['rsv.started-service'] = True
core.state['rsv.running-service'] = True
def test_02_stop_xrootd(self):
if core.state['xrootd.backups-exist']:
files.restore(core.config['xrootd.config'], "xrootd")
files.restore(core.config['xrootd.logging-config'], "xrootd")
files.restore(core.config['xrootd.authfile'], "xrootd")
files.restore(xrootd.logfile("standalone"), "xrootd", ignore_missing=True)
if "SCITOKENS" in core.config['xrootd.security']:
files.restore('/etc/xrootd/scitokens.conf', "xrootd")
files.remove("/etc/xrootd/config.d/99-osgtest-ztn.cfg", force=True)
if os.path.exists(xrootd.ROOTDIR):
shutil.rmtree(xrootd.ROOTDIR)
# Get xrootd service back to its original state
self.skip_ok_unless(core.state['xrootd.is-configured'], "xrootd is not configured")
xrootd_service = core.config['xrootd_service']
if service.is_running(xrootd_service):
service.check_stop(xrootd_service, force=True)
if core.state.get('xrootd.service-was-running', False):
service.check_start(xrootd_service, force=True)
def test_02_start_slurmdbd(self):
core.state['slurmdbd.started-service'] = False
core.state['slurmdbd.ready'] = False
self.slurm_reqs()
self.skip_bad_unless(mysql.is_running(), 'slurmdbd requires mysql')
core.config['slurmdbd.config'] = os.path.join(
core.config['slurm.config-dir'], 'slurmdbd.conf')
core.config['slurmdbd.user'] = "******"
core.config['slurmdbd.name'] = "osg_test_slurmdb"
mysql.check_execute(
"create database %s; " % core.config['slurmdbd.name'],
'create slurmdb')
mysql.check_execute("create user %s; " % core.config['slurmdbd.user'],
'add slurmdb user')
mysql.check_execute(
"grant usage on *.* to %s; " % core.config['slurmdbd.user'],
'slurmdb user access')
mysql.check_execute(
"grant all privileges on %s.* to %s identified by '%s'; " %
(core.config['slurmdbd.name'], core.config['slurmdbd.user'],
core.options.password), 'slurmdb user permissions')
mysql.check_execute("flush privileges;", 'reload privileges')
files.write(core.config['slurmdbd.config'],
SLURMDBD_CONFIG.format(
name=core.config['slurmdbd.name'],
user=core.config['slurmdbd.user'].split('\'')[1],
password=core.options.password,
port=mysql.PORT),
owner='slurm',
chmod=0o644)
stat = core.get_stat(SLURMDBD_LOG)
service.check_start('slurmdbd')
sentinel = core.monitor_file(SLURMDBD_LOG, stat,
'slurmdbd version.+started', 30.0)
if sentinel:
core.state['slurmdbd.ready'] = True
# Adding the cluster to the database
command = ('sacctmgr', '-i', 'add', 'cluster', CLUSTER_NAME)
core.check_system(command, 'add slurm cluster')
def test_01_start_condor(self):
core.state['condor.started-service'] = False
core.state['condor.running-service'] = False
core.skip_ok_unless_installed('condor')
core.config['condor.collectorlog'] = condor.config_val('COLLECTOR_LOG')
if service.is_running('condor'):
core.state['condor.running-service'] = True
return
config_dirs = re.split(r'[, ]+', condor.config_val('LOCAL_CONFIG_DIR'))
core.config['condor.personal_condor'] = join(config_dirs[-1], '99-personal-condor.conf')
files.write(core.config['condor.personal_condor'], personal_condor_config, owner='condor', chmod=0o644)
core.config['condor.collectorlog_stat'] = core.get_stat(core.config['condor.collectorlog'])
service.check_start('condor')
core.state['condor.started-service'] = True
core.state['condor.running-service'] = True
def test_04_start_condorce(self):
if core.el_release() >= 7:
core.config['condor-ce.lockfile'] = '/var/lock/condor-ce/htcondor-ceLock'
else:
core.config['condor-ce.lockfile'] = '/var/lock/subsys/condor-ce'
core.state['condor-ce.started-service'] = False
core.state['condor-ce.schedd-ready'] = False
core.skip_ok_unless_installed('condor', 'htcondor-ce', 'htcondor-ce-client')
core.config['condor-ce.collectorlog'] = condor.ce_config_val('COLLECTOR_LOG')
if service.is_running('condor-ce'):
core.state['condor-ce.schedd-ready'] = True
self.skip_ok('already running')
service.check_start('condor-ce')
stat = core.get_stat(core.config['condor-ce.collectorlog'])
if condor.wait_for_daemon(core.config['condor-ce.collectorlog'], stat, 'Schedd', 300.0):
core.state['condor-ce.schedd-ready'] = True
def test_03_start_slurm(self):
core.config['slurm.service-name'] = 'slurm'
if core.el_release() == 7:
core.config['slurm.service-name'] += 'd'
core.config['slurm.ctld-service-name'] = 'slurmctld'
core.state['%s.started-service' % core.config['slurm.service-name']] = False
self.slurm_reqs()
self.skip_ok_if(service.is_running(core.config['slurm.service-name']), 'slurm already running')
stat = core.get_stat(CTLD_LOG)
if core.el_release() == 7:
# slurmctld is handled by /etc/init.d/slurm on EL6
command = ['slurmctld']
core.check_system(command, 'enable slurmctld')
service.check_start(core.config['slurm.service-name'])
service.check_start(core.config['slurm.ctld-service-name'])
else:
service.check_start(core.config['slurm.service-name'])
core.monitor_file(CTLD_LOG,
stat,
'slurm_rpc_node_registration complete for %s' % SHORT_HOSTNAME,
60.0)
log_stat = core.get_stat(SLURM_LOG)
core.monitor_file(SLURM_LOG,
log_stat,
'slurmd started',
60.0)
command = ['scontrol', 'update', 'nodename=%s' % SHORT_HOSTNAME, 'state=idle']
core.check_system(command, 'enable slurm node')
def test_08_start_xrootd(self):
self.skip_ok_unless(core.state['xrootd.is-configured'], "xrootd is not configured")
if core.config['xrootd.multiuser']:
core.config['xrootd_service'] = "xrootd-privileged@standalone"
else:
core.config['xrootd_service'] = "xrootd@standalone"
core.state['xrootd.service-was-running'] = False
# Stop the service so it gets our new config
if service.is_running(core.config['xrootd_service']):
core.state['xrootd.service-was-running'] = True
service.stop(core.config['xrootd_service'], force=True)
time.sleep(5)
# clear the logfile so it only contains our run
if core.options.manualrun:
files.preserve_and_remove(xrootd.logfile("standalone"), "xrootd")
try:
service.check_start(core.config['xrootd_service'], min_up_time=5)
except Exception:
xrootd.dump_log(125, "standalone")
raise
def test_02_start_slurmdbd(self):
core.state['slurmdbd.started-service'] = False
self.slurm_reqs()
core.skip_ok_unless_installed('slurm-slurmdbd')
self.skip_bad_unless(mysql.is_running(), 'slurmdbd requires mysql')
core.config['slurmdbd.config'] = os.path.join(
core.config['slurm.config-dir'], 'slurmdbd.conf')
core.config['slurmdbd.user'] = "******"
core.config['slurmdbd.name'] = "osg_test_slurmdb"
mysql.check_execute(
"create database %s; " % core.config['slurmdbd.name'],
'create slurmdb')
mysql.check_execute("create user %s; " % core.config['slurmdbd.user'],
'add slurmdb user')
mysql.check_execute(
"grant usage on *.* to %s; " % core.config['slurmdbd.user'],
'slurmdb user access')
mysql.check_execute(
"grant all privileges on %s.* to %s identified by '%s'; " %
(core.config['slurmdbd.name'], core.config['slurmdbd.user'],
core.options.password), 'slurmdb user permissions')
mysql.check_execute("flush privileges;", 'reload privileges')
db_config_vals = {
'name': core.config['slurmdbd.name'],
'user': core.config['slurmdbd.user'].split('\'')[1],
'pass': core.options.password
}
files.write(core.config['slurmdbd.config'],
SLURMDBD_CONFIG % db_config_vals,
owner='slurm',
chmod=0o644)
service.check_start('slurmdbd')
# Adding the cluster to the database
command = ('sacctmgr', '-i', 'add', 'cluster', CLUSTER_NAME)
core.check_system(command, 'add slurm cluster')
def test_05_start_pbs(self):
core.state['pbs_server.started-service'] = False
core.state['torque.nodes-up'] = False
core.skip_ok_unless_installed(*self.required_rpms, by_dependency=True)
self.skip_bad_unless(service.is_running('trqauthd'),
'pbs_server requires trqauthd')
self.skip_ok_if(service.is_running('pbs_server'),
'pbs server already running')
server_log = '/var/log/torque/server_logs/' + date.today().strftime(
'%Y%m%d')
try:
server_log_stat = os.stat(server_log)
except OSError:
server_log_stat = None
service.check_start('pbs_server')
# Wait until the server is up before writing the rest of the config
core.monitor_file(server_log, server_log_stat, '.*Server Ready.*',
60.0)
core.check_system("echo '%s' | qmgr %s" %
(self.pbs_config, core.get_hostname()),
"Configuring pbs server",
shell=True)
# wait up to 5 minutes for the server to recognize the node
start_time = time.time()
while (time.time() - start_time) < 600:
command = ('/usr/bin/qnodes', '-s', core.get_hostname())
stdout, _, fail = core.check_system(command, 'Get pbs node info')
self.assert_(stdout.find('error') == -1, fail)
if stdout.find('state = free'):
core.state['torque.nodes-up'] = True
break
if not core.state['torque.nodes-up']:
self.fail('PBS nodes not coming up')
def test_05_start_bestman(self):
core.config['bestman.pid-file'] = '/var/run/bestman2.pid'
core.state['bestman.started-server'] = False
core.state['bestman.server-running'] = False
core.skip_ok_unless_installed('bestman2-server', 'bestman2-client', 'gums-service')
if service.is_running('bestman2'):
core.state['bestman.server-running'] = True
self.skip_ok('bestman2 already running')
# Dump the bestman logs into the test logs for debugging
def _dump_logfiles():
logdir = '/var/log/bestman2'
for logfile in ('bestman2.log', 'event.srm.log'):
core.system(('cat', os.path.join(logdir, logfile)))
try:
service.check_start('bestman2')
except AssertionError:
_dump_logfiles()
raise
core.state['bestman.started-server'] = True
core.state['bestman.server-running'] = True
def test_05_start_bestman(self):
core.config['bestman.pid-file'] = '/var/run/bestman2.pid'
core.state['bestman.started-server'] = False
core.state['bestman.server-running'] = False
core.skip_ok_unless_installed('bestman2-server', 'bestman2-client', 'gums-service')
if service.is_running('bestman2'):
core.state['bestman.server-running'] = True
self.skip_ok('bestman2 already running')
# Dump the bestman logs into the test logs for debugging
def _dump_logfiles():
logdir = '/var/log/bestman2'
for logfile in ('bestman2.log', 'event.srm.log'):
core.system(('cat', os.path.join(logdir, logfile)))
try:
service.check_start('bestman2')
except AssertionError:
_dump_logfiles()
raise
core.state['bestman.started-server'] = True
core.state['bestman.server-running'] = True
def test_05_start_tomcat(self):
core.skip_ok_unless_installed(tomcat.pkgname())
core.state['tomcat.started'] = False
catalina_log = tomcat.catalinafile()
initial_stat = core.get_stat(catalina_log)
tomcat_sentinel = r'Server startup in \d+ ms'
# Bump log level
core.config['tomcat.logging-conf'] = os.path.join(
tomcat.sysconfdir(), 'logging.properties')
files.append(core.config['tomcat.logging-conf'],
'org.apache.catalina.level = %s\n' % 'FINER',
owner='tomcat',
backup=True)
old_str = "1catalina.org.apache.juli.FileHandler.prefix = catalina."
repl_str = ("1catalina.org.apache.juli.FileHandler.prefix = catalina\n"
"1catalina.org.apache.juli.FileHandler.rotatable = false")
files.replace(core.config['tomcat.logging-conf'],
old_str,
repl_str,
owner='tomcat',
backup=False)
service.check_start(tomcat.pkgname())
if core.options.nightly:
timeout = 3600.0
else:
timeout = 1200.0
line, gap = core.monitor_file(catalina_log, initial_stat,
tomcat_sentinel, timeout)
self.assert_(
line is not None, 'Tomcat did not start within the %d min window' %
int(timeout / 60))
core.state['tomcat.started'] = True
core.log_message('Tomcat started after %.1f seconds' % gap)
def test_02_start_origin(self):
if not service.is_running("xrootd@stashcache-origin-server"):
service.check_start("xrootd@stashcache-origin-server")
def test_start_haveged(self):
core.skip_ok_unless_installed('haveged')
service.check_start('haveged')
def test_07_ping_with_gums(self):
core.state['condor-ce.gums-auth'] = False
self.general_requirements()
core.skip_ok_unless_installed('gums-service')
# Setting up GUMS auth using the instructions here:
# https://opensciencegrid.github.io/docs/compute-element/install-htcondor-ce/#authentication-with-gums
hostname = core.get_hostname()
lcmaps_contents = '''gumsclient = "lcmaps_gums_client.mod"
"-resourcetype ce"
"-actiontype execute-now"
"-capath /etc/grid-security/certificates"
"-cert /etc/grid-security/hostcert.pem"
"-key /etc/grid-security/hostkey.pem"
"--cert-owner root"
# Change this URL to your GUMS server
"--endpoint https://%s:8443/gums/services/GUMSXACMLAuthorizationServicePort"
verifyproxy = "lcmaps_verify_proxy.mod"
"--allow-limited-proxy"
" -certdir /etc/grid-security/certificates"
# lcmaps policies require at least two modules, so these are here to
# fill in if only one module is needed. "good | bad" has no effect.
good = "lcmaps_dummy_good.mod"
bad = "lcmaps_dummy_bad.mod"
authorize_only:
## Policy 1: GUMS but not SAZ (most common, default)
gumsclient -> good | bad
''' % hostname
gums_properties_contents = '''gums.location=https://%s:8443/gums/services/GUMSAdmin
gums.authz=https://%s:8443/gums/services/GUMSXACMLAuthorizationServicePort
''' % (hostname, hostname)
core.config['condor-ce.lcmapsdb'] = '/etc/lcmaps.db'
core.config['condor-ce.gums-properties'] = '/etc/gums/gums-client.properties'
core.config['condor-ce.gsi-authz'] = '/etc/grid-security/gsi-authz.conf'
files.write(core.config['condor-ce.lcmapsdb'], lcmaps_contents, owner='condor-ce.gums')
files.write(core.config['condor-ce.gums-properties'], gums_properties_contents, owner='condor-ce')
files.replace(core.config['condor-ce.gsi-authz'],
'# globus_mapping liblcas_lcmaps_gt4_mapping.so lcmaps_callout',
'globus_mapping liblcas_lcmaps_gt4_mapping.so lcmaps_callout',
owner='condor-ce')
try:
core.state['condor-ce.gums-auth'] = True
service.check_stop('condor-ce')
stat = core.get_stat(core.config['condor-ce.collectorlog'])
service.check_start('condor-ce')
# Wait for the schedd to come back up
self.failUnless(condor.wait_for_daemon(core.config['condor-ce.collectorlog'], stat, 'Schedd', 300.0),
'Schedd failed to restart within the 1 min window')
command = ('condor_ce_ping', 'WRITE', '-verbose')
stdout, _, _ = core.check_system(command, 'ping using GSI and gridmap', user=True)
self.assert_(re.search(r'Authorized:\s*TRUE', stdout), 'could not authorize with GSI')
finally:
files.restore(core.config['condor-ce.lcmapsdb'], 'condor-ce.gums')
files.restore(core.config['condor-ce.gsi-authz'], 'condor-ce')
files.restore(core.config['condor-ce.gums-properties'], 'condor-ce')
def test_03_start(self):
core.state['gsisshd.started-service'] = False
self.skip_ok_unless(core.state['gsisshd.can-run'], "Can't run gsisshd (see above)")
service.check_start('gsisshd')
def test_03_start_cache(self):
if not service.is_running("xrootd@stashcache-cache-server"):
service.check_start("xrootd@stashcache-cache-server")
def test_02_start_pbs_sched(self):
core.state['pbs_sched.started-service'] = False
core.skip_ok_unless_installed(*self.required_rpms, by_dependency=True)
self.skip_ok_if(service.is_running('pbs_sched'),
'PBS sched already running')
service.check_start('pbs_sched')
def test_07_ping_with_gums(self):
core.state['condor-ce.gums-auth'] = False
self.general_requirements()
core.skip_ok_unless_installed('gums-service')
# Setting up GUMS auth using the instructions here:
# https://opensciencegrid.github.io/docs/compute-element/install-htcondor-ce/#authentication-with-gums
hostname = core.get_hostname()
lcmaps_contents = '''gumsclient = "lcmaps_gums_client.mod"
"-resourcetype ce"
"-actiontype execute-now"
"-capath /etc/grid-security/certificates"
"-cert /etc/grid-security/hostcert.pem"
"-key /etc/grid-security/hostkey.pem"
"--cert-owner root"
# Change this URL to your GUMS server
"--endpoint https://%s:8443/gums/services/GUMSXACMLAuthorizationServicePort"
verifyproxy = "lcmaps_verify_proxy.mod"
"--allow-limited-proxy"
" -certdir /etc/grid-security/certificates"
# lcmaps policies require at least two modules, so these are here to
# fill in if only one module is needed. "good | bad" has no effect.
good = "lcmaps_dummy_good.mod"
bad = "lcmaps_dummy_bad.mod"
authorize_only:
## Policy 1: GUMS but not SAZ (most common, default)
gumsclient -> good | bad
''' % hostname
gums_properties_contents = '''gums.location=https://%s:8443/gums/services/GUMSAdmin
gums.authz=https://%s:8443/gums/services/GUMSXACMLAuthorizationServicePort
''' % (hostname, hostname)
core.config['condor-ce.lcmapsdb'] = '/etc/lcmaps.db'
core.config['condor-ce.gums-properties'] = '/etc/gums/gums-client.properties'
core.config['condor-ce.gsi-authz'] = '/etc/grid-security/gsi-authz.conf'
files.write(core.config['condor-ce.lcmapsdb'], lcmaps_contents, owner='condor-ce.gums')
files.write(core.config['condor-ce.gums-properties'], gums_properties_contents, owner='condor-ce')
files.replace(core.config['condor-ce.gsi-authz'],
'# globus_mapping liblcas_lcmaps_gt4_mapping.so lcmaps_callout',
'globus_mapping liblcas_lcmaps_gt4_mapping.so lcmaps_callout',
owner='condor-ce')
try:
core.state['condor-ce.gums-auth'] = True
service.check_stop('condor-ce')
stat = core.get_stat(core.config['condor-ce.collectorlog'])
service.check_start('condor-ce')
# Wait for the schedd to come back up
self.failUnless(condor.wait_for_daemon(core.config['condor-ce.collectorlog'], stat, 'Schedd', 300.0),
'Schedd failed to restart within the 1 min window')
command = ('condor_ce_ping', 'WRITE', '-verbose')
stdout, _, _ = core.check_system(command, 'ping using GSI and gridmap', user=True)
self.assert_(re.search(r'Authorized:\s*TRUE', stdout), 'could not authorize with GSI')
finally:
files.restore(core.config['condor-ce.lcmapsdb'], 'condor-ce.gums')
files.restore(core.config['condor-ce.gsi-authz'], 'condor-ce')
files.restore(core.config['condor-ce.gums-properties'], 'condor-ce')
def start_xrootd(instance):
svc = "xrootd@%s" % instance
if not service.is_running(svc):
service.check_start(svc)
def test_02_start_pbs_sched(self):
core.state['pbs_sched.started-service'] = False
core.skip_ok_unless_installed(*self.required_rpms, by_dependency=True)
self.skip_ok_if(service.is_running('pbs_sched'), 'PBS sched already running')
service.check_start('pbs_sched')
def test_03_start(self):
core.state['gsisshd.started-service'] = False
self.skip_ok_unless(core.state['gsisshd.can-run'],
"Can't run gsisshd (see above)")
service.check_start('gsisshd')
