def analyze(self, datadir, resultdir, noreanalyze):
if not os.path.isdir(datadir):
logging.debug('"%s" does not exist. Skipping', datadir)
return {}
etce_stores = []
for dirname, _, filenames in os.walk(datadir):
if 'etce.store' in filenames:
etce_stores.append(os.path.join(dirname, 'etce.store'))
ws = WrapperStore(os.path.join(sorted(etce_stores)[0]))
store_values = list(ws.read().values())
starttimestr = store_values[0]['etce']['starttime']
dt = etce.timeutils.strtimetodatetime(starttimestr)
starttime = time.mktime(dt.timetuple())
mgr = AnalyzerManager(self._analyzerconfig)
# process individual data files
trialfilesmap = defaultdict(lambda: [])
for hostname in sorted(os.listdir(datadir)):
hostdir = os.path.join(datadir, hostname)
if not os.path.isdir(hostdir):
continue
for datafile in sorted(os.listdir(hostdir)):
# return a map where the key is the name of any analyzer that
# operates on datafile, and values are the node output files generated
# by that analyzer
resultfilesmap = mgr.analyzefile(
PathInfo(datadir, hostname, datafile), resultdir,
noreanalyze)
for analyzer, resultfiles in resultfilesmap.items():
trialfilesmap[analyzer].extend(resultfiles)
# then combine the results
combinedfiles = defaultdict(lambda: None)
for analyzer, trialfiles in trialfilesmap.items():
if len(trialfiles) == 0:
continue
combinedfiles[analyzer] = mgr.combinetrialresults(
trialfiles, resultdir, analyzer, starttime)
return combinedfiles
def __init__(self, wrappername, wrapperinstance, trialargs, testargs,
config, testdir):
self._trialargs = trialargs
self._testargs = testargs
self._config = config
self._testdir = testdir
self._platform = Platform()
self._wrappername = wrappername
self._sudo = False
self._default_pidfilename = \
'%s/etce.%s.%s.pid' \
% (os.path.join(self._config.get('etce', 'WORK_DIRECTORY'), 'lock'),
self.platform.hostname(),
self._wrappername)
self._description = wrapperinstance.__doc__
# start with reserved args set here ...
self._args = {
'default_pidfilename': self._default_pidfilename,
'nodename': self._testdir.nodename(),
'nodeid': self._testdir.nodeid(),
'testname': self._testdir.name(),
'wrappername': self._wrappername,
'infile': None,
'outfile': None
}
# ... and the ones passed in
self._args.update(trialargs)
# these are the reserved args that cannot be overwritten
self._reserved_args = set(self._args)
# fill in the arguments registered by the wrapper
wrapperinstance.register(self)
storefile = os.path.join(self._trialargs['logdirectory'], 'etce.store')
self._wrapperstore = WrapperStore(storefile)
self._wrapperstore.update(
{'etce': {
'starttime': self._trialargs['starttime']
}}, self._args['nodename'])
class WrapperContextImpl(ArgRegistrar):
''' WrapperContextImpl implements the WrapperContext interface.'''
def __init__(self, wrappername, wrapperinstance, trialargs, testargs,
config, testdir):
self._trialargs = trialargs
self._testargs = testargs
self._config = config
self._testdir = testdir
self._platform = Platform()
self._wrappername = wrappername
self._sudo = False
self._default_pidfilename = '%s/etce.%s.%s.pid' \
% (os.path.join(self._config.get('etce', 'WORK_DIRECTORY'), 'lock'),
self.platform.hostname(),
self._wrappername)
self._description = wrapperinstance.__doc__
# start with reserved args set here ...
self._args = {
'default_pidfilename': self._default_pidfilename,
'nodename': self._testdir.nodename(),
'nodeid': self._testdir.nodeid(),
'testname': self._testdir.name(),
'wrappername': self._wrappername,
'infile': None,
'outfile': None
}
# ... and the ones passed in
self._args.update(trialargs)
# these are the reserved args that cannot be overwritten
self._reserved_args = set(self._args)
# fill in the arguments registered by the wrapper
wrapperinstance.register(self)
storefile = os.path.join(self._trialargs['logdirectory'], 'etce.store')
self._wrapperstore = WrapperStore(storefile)
self._wrapperstore.update(
{'etce': {
'starttime': self._trialargs['starttime']
}}, self._args['nodename'])
def register_argument(self, argname, defaultval, description):
if argname in self._reserved_args:
raise ValueError('Wrapper "%s" attempting to register a ' \
'reserved argument "%s". Quitting.' % \
(self._args['wrappername'],
argname))
if self._testdir.hasconfig(self._wrappername, argname):
self._args[argname] = \
self._testdir.getconfig(self._wrappername, argname, defaultval)
elif argname in self._testargs:
self._args[argname] = self._testargs[argname]
else:
self._args[argname] = defaultval
def register_infile_name(self, name):
self._args['infile'] = self._testdir.getfile(name)
def register_outfile_name(self, name):
self._args['outfile'] = os.path.join(self._trialargs['logdirectory'],
name)
def run_with_sudo(self):
# ignore run with sudo requests when configured to do so
if self._config.get('etce', 'IGNORE_RUN_WITH_SUDO').lower() == 'yes':
return
self._sudo = True
def store(self, namevaldict):
self._wrapperstore.update({self._args['wrappername']: namevaldict},
self._args['nodename'])
@property
def platform(self):
return self._platform
@property
def args(self):
return ArgProxy(self._args)
def daemonize(self,
command,
argstr,
stdout=None,
stderr=None,
pidfilename=None,
genpidfile=True,
pidincrement=0,
starttime=None,
extra_paths=[]):
commandstr = self._build_commandstr(command, argstr, extra_paths)
# print the commandstr and return on a dryrun
if self._trialargs['dryrun']:
print(commandstr)
sys.stdout.flush()
return
# 1. call self.stop(pidfilename)
self.stop(pidfilename)
# run the command
pid, subprocess = etce.utils.daemonize_command(commandstr, stdout,
stderr, starttime)
# return on parent
if pid > 0:
return
# 2. if genpidfile is True, and pidfilename is None,
# generate the pidfilename
if genpidfile and pidfilename is None:
pidfilename = self._default_pidfilename
# 3. write the pid to pidfilename
if genpidfile:
with open(pidfilename, 'w') as pidfile:
pidfile.write(str(subprocess.pid + pidincrement))
# 4. wait on subprocess
subprocess.wait()
# 5. exit, do not return, because returning
# will cause any subsequent wrappers in this
# step to be rerun
sys.exit(0)
def run(self,
command,
argstr,
stdout=None,
stderr=None,
pidfilename=None,
genpidfile=True,
pidincrement=0,
extra_paths=[]):
commandstr = self._build_commandstr(command, argstr, extra_paths)
# print the commandstr and return on a dryrun
if self._trialargs['dryrun']:
print(commandstr)
sys.stdout.flush()
return
self.stop(pidfilename)
print(commandstr)
sys.stdout.flush()
stdoutfd = None
stderrfd = None
if not stdout is None:
stdoutfd = open(stdout, 'w')
if not stderr is None:
if stdout == stderr:
stderrfd = stdoutfd
else:
stderrfd = open(stderr, 'w')
# generate the pidfilename
if genpidfile and pidfilename is None:
pidfilename = self._default_pidfilename
# create the Popen subprocess
sp = subprocess.Popen(shlex.split(commandstr),
stdout=stdoutfd,
stderr=stderrfd)
# write the pid to pidfilename
if genpidfile:
with open(pidfilename, 'w') as pidfile:
pidfile.write(str(sp.pid + pidincrement))
# wait on subprocess
sp.wait()
def stop(self, pidfilename=None, signal=signal.SIGQUIT, sudo=True):
# use default pidfilename if None specified
if pidfilename is None:
pidfilename = self._default_pidfilename
# if found a pid, kill the process and remove the file
self._platform.kill(pidfilename, signal, sudo)
def _build_commandstr(self, command, argstr, extra_paths):
all_paths = os.environ['PATH'].split(':') + list(extra_paths)
existing_paths = filter(os.path.isdir, all_paths)
found_paths = []
for existing_path in existing_paths:
if command in os.listdir(existing_path):
found_paths.append(existing_path)
if not found_paths:
raise WrapperError('Cannot find command "%s" in system paths {%s}. Quitting.' \
% (command, ','.join(all_paths)))
commandstr = ' '.join([os.path.join(found_paths[0], command), argstr])
# run with sudo if wrapper requested it
if self._sudo:
commandstr = 'sudo ' + commandstr
return commandstr
class WrapperContextImpl(ArgRegistrar):
''' WrapperContextImpl implements the WrapperContext interface.'''
def __init__(self, wrappername, wrapperinstance, trialargs, testargs,
config, testdir):
self._trialargs = trialargs
self._testargs = testargs
self._config = config
self._testdir = testdir
self._platform = Platform()
self._wrappername = wrappername
self._default_pidfilename = '%s/etce.%s.%s.pid' \
% (os.path.join(self._config.get('etce', 'WORK_DIRECTORY'), 'lock'),
self.platform.hostname(),
self._wrappername)
self._description = wrapperinstance.__doc__
# start with reserved args set here ...
self._args = {
'default_pidfilename': self._default_pidfilename,
'nodename': self._testdir.nodename(),
'nodeid': self._testdir.nodeid(),
'testname': self._testdir.name(),
'wrappername': self._wrappername,
'infile': None,
'outfile': None
}
# ... and the ones passed in
self._args.update(trialargs)
# these are the reserved args that cannot be overwritten
self._reserved_args = set(self._args)
# fill in the arguments registered by the wrapper
wrapperinstance.register(self)
storefile = os.path.join(self._trialargs['logdirectory'], 'etce.store')
self._wrapperstore = WrapperStore(storefile)
self._wrapperstore.update(
{'etce': {
'starttime': self._trialargs['starttime']
}}, self._args['nodename'])
def register_argument(self, argname, defaultval, description):
if argname in self._reserved_args:
raise ValueError('Wrapper "%s" attempting to register a ' \
'reserved argument "%s". Quitting.' % \
(self._args['wrappername'],
argname))
if self._testdir.hasconfig(self._wrappername, argname):
self._args[argname] = \
self._testdir.getconfig(self._wrappername, argname, defaultval)
elif argname in self._testargs:
self._args[argname] = self._testargs[argname]
else:
self._args[argname] = defaultval
def register_infile_name(self, name):
self._args['infile'] = self._testdir.getfile(name)
def register_outfile_name(self, name):
self._args['outfile'] = os.path.join(self._trialargs['logdirectory'],
name)
def store(self, namevaldict):
self._wrapperstore.update({self._args['wrappername']: namevaldict},
self._args['nodename'])
@property
def platform(self):
return self._platform
@property
def args(self):
return ArgProxy(self._args)
def daemonize(self,
commandstr,
stdout=None,
stderr=None,
pidfilename=None,
genpidfile=True,
pidincrement=0,
starttime=None):
# print the commandstr and return on a dryrun
if self._trialargs['dryrun']:
print commandstr
return
# 1. call self.stop(pidfilename)
self.stop(pidfilename)
# run the command
pid,subprocess = \
etce.utils.daemonize_command(commandstr,
stdout,
stderr,
starttime)
# return on parent
if pid > 0:
return
# 2. if genpidfile is True, and pidfilename is None,
# generate the pidfilename
if genpidfile and pidfilename is None:
pidfilename = self._default_pidfilename
# 3. write the pid to pidfilename
if genpidfile:
with open(pidfilename, 'w') as pidfile:
pidfile.write(str(subprocess.pid + pidincrement))
# 4. wait on subprocess
subprocess.wait()
# 5. exit, do not return, because returning
# will cause any subsequent wrappers in this
# step to be rerun
sys.exit(0)
def run(self,
commandstr,
stdout=None,
stderr=None,
pidfilename=None,
genpidfile=True,
pidincrement=0):
# print the commandstr and return on a dryrun
if self._trialargs['dryrun']:
print commandstr
return
self.stop(pidfilename)
print commandstr
command = shlex.split(commandstr)
stdoutfd = None
stderrfd = None
if not stdout is None:
stdoutfd = open(stdout, 'w')
if not stderr is None:
if stdout == stderr:
stderrfd = stdoutfd
else:
stderrfd = open(stderr, 'w')
# generate the pidfilename
if genpidfile and pidfilename is None:
pidfilename = self._default_pidfilename
# create the Popen subprocess
sp = subprocess.Popen(command, stdout=stdoutfd, stderr=stderrfd)
# write the pid to pidfilename
if genpidfile:
with open(pidfilename, 'w') as pidfile:
pidfile.write(str(sp.pid + pidincrement))
# wait on subprocess
sp.wait()
def stop(self, pidfilename=None, signal=signal.SIGQUIT, decorator=''):
# use default pidfilename if None specified
if pidfilename is None:
pidfilename = self._default_pidfilename
pid = self._platform.readpid(pidfilename)
# if found a pid, kill the process and remove the file
if pid:
try:
print 'killing pid %d found in %s' % (pid, pidfilename)
command = '%s kill -%d %d' % (pid, signal)
sp = subprocess.Popen(command)
sp.wait()
#os.kill(pid, signal.SIGQUIT)
except OSError as e:
# orphaned pidfile - process already dead
pass
finally:
os.remove(pidfilename)