def spawn(controller, backend, task_info, env, args):
task_env = dict(env)
# 1. set env.variables
# BEAH_THOST - host name
# BEAH_TPORT - port
# BEAH_TSOCKET - socket
# BEAH_TID - id of task - used to introduce itself when opening socket
task_id = task_info['id']
conf = config.get_conf('beah')
env_file = os.path.join(conf.get('TASK', 'VAR_ROOT'),
ENV_PATHNAME_TEMPLATE % task_id)
dict_update(
task_env,
CALLED_BY_BEAH="1",
BEAH_THOST=str(host),
BEAH_TPORT=str(port),
BEAH_TSOCKET=str(socket),
BEAH_TID=str(task_id),
BEAH_ROOT=conf.get('TASK', 'ROOT'),
BEAH_ENV=env_file,
)
ll = conf.get('TASK', 'LOG')
task_env.setdefault('BEAH_TASK_LOG', ll)
task_env.setdefault('BEAH_TASK_CONSOLE',
conf.get('TASK', 'CONSOLE_LOG', 'False'))
task_env.setdefault('TERM', 'dumb')
val = os.getenv('PYTHONPATH')
if val:
task_env['PYTHONPATH'] = val
jsonenv.export_env(env_file, task_env)
# 2. spawn a task
protocol = (proto or TaskStdoutProtocol)(task_id)
protocol.controller = controller
log.debug('spawn: Environment: %r.', task_env)
reactor.spawnProcess(protocol,
task_info['file'],
args=[task_info['file']] + (args or []),
env=task_env)
# FIXME: send an answer to backend(?)
return protocol.task_protocol
def spawn(controller, backend, task_info, env, args):
task_env = dict(env)
# 1. set env.variables
# BEAH_THOST - host name
# BEAH_TPORT - port
# BEAH_TSOCKET - socket
# BEAH_TID - id of task - used to introduce itself when opening socket
task_id = task_info['id']
conf = config.get_conf('beah')
env_file = os.path.join(conf.get('TASK', 'VAR_ROOT'),
ENV_PATHNAME_TEMPLATE % task_id)
dict_update(task_env,
CALLED_BY_BEAH="1",
BEAH_THOST=str(host),
BEAH_TPORT=str(port),
BEAH_TSOCKET=str(socket),
BEAH_TID=str(task_id),
BEAH_ROOT=conf.get('TASK', 'ROOT'),
BEAH_ENV=env_file,
)
ll = conf.get('TASK', 'LOG')
task_env.setdefault('BEAH_TASK_LOG', ll)
task_env.setdefault('BEAH_TASK_CONSOLE', conf.get('TASK', 'CONSOLE_LOG', 'False'))
task_env.setdefault('TERM', 'dumb')
val = os.getenv('PYTHONPATH')
if val:
task_env['PYTHONPATH'] = val
jsonenv.export_env(env_file, task_env)
# 2. spawn a task
protocol = (proto or TaskStdoutProtocol)(task_id)
protocol.controller = controller
log.debug('spawn: Environment: %r.', task_env)
reactor.spawnProcess(protocol, task_info['file'],
args=[task_info['file']]+(args or []), env=task_env)
# FIXME: send an answer to backend(?)
return protocol.task_protocol
def __init__(self, task_path, env):
self.process = None
self.listener = None
self.task_path = task_path
self.__done = False
self.__waits_for = []
# FIXME: is inheriting the whole environment desirable?
if env is not USE_DEFAULT:
self.env = dict(env)
else:
self.env = dict(os.environ)
# FIXME: Any other env.variables to set?
# FIXME: What values should be used here?
# - some values could be received from LC when task is scheduled, but
# it would create a dependency!
# - let's use fake values, and let the Backend translate it (if
# supported)
# - e.g. JOBID, RECIPESETID, RECIPEID are not interesting at all
# - use task_id for RECIPESETID, and BE (or LC eventually) should
# be able to find about the rest...
taskid = "J%(JOBID)s-S%(RECIPESETID)s-R%(RECIPEID)s-T%(TASKID)s" % self.env
# FIXME! use tempfile and upload log when process ends.
log = logging.getLogger('rhts_task')
twmisc.twisted_logging(log, level=logging.WARNING)
ll = self.env.get('BEAH_TASK_LOG', "warning")
log.setLevel(str2log_level(ll))
make_log_handler(log,
LOG_PATH,
"rhts_task_%s.log" % (taskid, ),
syslog=True,
console=self.env.get('BEAH_TASK_CONSOLE', False))
# parse task's metadata:
try:
from rhts import testinfo
ti = testinfo.parse_file(os.path.join(self.env['TESTPATH'],
'testinfo.desc'),
raise_errors=False)
except:
log.error("Error in tasks metadata: %s" % format_exc())
ti = None
if ti is not None:
for k, v in getattr(ti, 'environment', {}).iteritems():
self.env.setdefault(k, v)
for o in getattr(ti, 'options', []):
opt_lower = o.lower()
if opt_lower[0] == '-':
opt_lower = opt_lower[1:]
value = ''
else:
value = 'yes'
if opt_lower.startswith('compatible'):
self.env.setdefault('RHTS_OPTION_COMPATIBLE', value)
elif opt_lower.startswith('compatservice'):
self.env.setdefault('RHTS_OPTION_COMPAT_SERVICE', value)
elif opt_lower.startswith('strongeravc'):
self.env.setdefault('RHTS_OPTION_STRONGER_AVC', value)
# update log level if necessary:
ll2 = self.env.get('BEAH_TASK_LOG', ll)
if ll2 != ll:
log.setLevel(str2log_level(ll2))
# No point in storing everything in one big file. Use one file per task
rt = runtimes.ShelveRuntime(RUNTIME_PATHNAME_TEMPLATE % taskid)
self.__files = runtimes.TypeDict(rt, 'files')
# FIXME: use configurable range of ports.
self.variables = runtimes.TypeDict(rt, 'variables')
port = self.variables.setdefault(
'port', int(self.env.get('RHTS_PORT', random.randint(7080, 7099))))
self.variables.setdefault('nohup', False)
self.variables.setdefault('has_result', False)
self.env.setdefault(
'DIGEST_METHOD', 'no_digest'
) # use no digests by default... Seems waste of time on localhost.
self.env.setdefault('TESTORDER', '123') # FIXME: More sensible default
# update defaults:
for k, v in self.ENV_DEFAULTS.iteritems():
self.env.setdefault(k, v)
# provide sensible defaults for selected system env.variables:
self.env.setdefault('HOME', '/root')
self.env.setdefault('LANG', 'en_US.UTF-8')
# FIXME: should any checks go here?
# e.g. does Makefile PURPOSE exist? try running `make testinfo.desc`? ...
self.controller = ControllerLink(self)
stdio.StandardIO(self.controller)
self.task = RHTSTask(self)
self.server = RHTSServer(self)
# If IPv6 has not been disabled, attempt to listen on IPv6
# otherwise fall back to IPv4
def listen_tcp(interface):
return reactor.listenTCP(port, self.server, interface=interface)
conf = beah.config.get_conf('beah')
if not parse_bool(conf.get('DEFAULT', 'IPV6_DISABLED')):
try:
listen_tcp('::1')
self.env['RESULT_SERVER'] = '[::1]:%s' % port
except CannotListenError:
listen_tcp('127.0.0.1')
self.env['RESULT_SERVER'] = '127.0.0.1:%s' % port
else:
listen_tcp('127.0.0.1')
self.env['RESULT_SERVER'] = '127.0.0.1:%s' % port
# save env:
env_file = ENV_PATHNAME_TEMPLATE % taskid
self.env['RHTS_ENV'] = env_file
jsonenv.export_env(env_file, self.env)
# Execute rhts-test-runner.sh
self.server_started()
def __init__(self, task_path, env):
self.process = None
self.listener = None
self.task_path = task_path
self.__done = False
self.__waits_for = []
# FIXME: is inheriting the whole environment desirable?
if env is not USE_DEFAULT:
self.env = dict(env)
else:
self.env = dict(os.environ)
# FIXME: Any other env.variables to set?
# FIXME: What values should be used here?
# - some values could be received from LC when task is scheduled, but
# it would create a dependency!
# - let's use fake values, and let the Backend translate it (if
# supported)
# - e.g. JOBID, RECIPESETID, RECIPEID are not interesting at all
# - use task_id for RECIPESETID, and BE (or LC eventually) should
# be able to find about the rest...
taskid = "J%(JOBID)s-S%(RECIPESETID)s-R%(RECIPEID)s-T%(TASKID)s" % self.env
# FIXME! use tempfile and upload log when process ends.
log = logging.getLogger('rhts_task')
twmisc.twisted_logging(log, level=logging.WARNING)
ll = self.env.get('BEAH_TASK_LOG', "warning")
log.setLevel(str2log_level(ll))
make_log_handler(log, LOG_PATH, "rhts_task_%s.log" % (taskid,),
syslog=True,
console=self.env.get('BEAH_TASK_CONSOLE', False))
# parse task's metadata:
try:
from rhts import testinfo
ti = testinfo.parse_file(os.path.join(self.env['TESTPATH'], 'testinfo.desc'), raise_errors=False)
except:
log.error("Error in tasks metadata: %s" % format_exc())
ti = None
if ti is not None:
for k,v in getattr(ti, 'environment', {}).iteritems():
self.env.setdefault(k, v)
for o in getattr(ti, 'options', []):
opt_lower = o.lower()
if opt_lower[0] == '-':
opt_lower = opt_lower[1:]
value = ''
else:
value = 'yes'
if opt_lower.startswith('compatible'):
self.env.setdefault('RHTS_OPTION_COMPATIBLE', value)
elif opt_lower.startswith('compatservice'):
self.env.setdefault('RHTS_OPTION_COMPAT_SERVICE', value)
elif opt_lower.startswith('strongeravc'):
self.env.setdefault('RHTS_OPTION_STRONGER_AVC', value)
# update log level if necessary:
ll2 = self.env.get('BEAH_TASK_LOG', ll)
if ll2 != ll:
log.setLevel(str2log_level(ll2))
# No point in storing everything in one big file. Use one file per task
rt = runtimes.ShelveRuntime(RUNTIME_PATHNAME_TEMPLATE % taskid)
self.__files = runtimes.TypeDict(rt, 'files')
# FIXME: use configurable range of ports.
self.variables = runtimes.TypeDict(rt, 'variables')
port = self.variables.setdefault('port', int(self.env.get('RHTS_PORT', random.randint(7080, 7099))))
self.variables.setdefault('nohup', False)
self.variables.setdefault('has_result', False)
self.env.setdefault('DIGEST_METHOD', 'no_digest') # use no digests by default... Seems waste of time on localhost.
self.env.setdefault('TESTORDER', '123') # FIXME: More sensible default
# update defaults:
for k, v in self.ENV_DEFAULTS.iteritems():
self.env.setdefault(k, v)
# provide sensible defaults for selected system env.variables:
self.env.setdefault('HOME', '/root')
self.env.setdefault('LANG', 'en_US.UTF-8')
# FIXME: should any checks go here?
# e.g. does Makefile PURPOSE exist? try running `make testinfo.desc`? ...
self.controller = ControllerLink(self)
stdio.StandardIO(self.controller)
self.task = RHTSTask(self)
self.server = RHTSServer(self)
# If IPv6 has not been disabled, attempt to listen on IPv6
# otherwise fall back to IPv4
def listen_tcp(interface):
return reactor.listenTCP(port, self.server,
interface=interface)
conf = beah.config.get_conf('beah')
if not parse_bool(conf.get('DEFAULT', 'IPV6_DISABLED')):
try:
listen_tcp('::1')
self.env['RESULT_SERVER'] = '[::1]:%s' % port
except CannotListenError:
listen_tcp('127.0.0.1')
self.env['RESULT_SERVER'] = '127.0.0.1:%s' % port
else:
listen_tcp('127.0.0.1')
self.env['RESULT_SERVER'] = '127.0.0.1:%s' % port
# save env:
env_file = ENV_PATHNAME_TEMPLATE % taskid
self.env['RHTS_ENV'] = env_file
jsonenv.export_env(env_file, self.env)
# Execute rhts-test-runner.sh
self.server_started()
def testMain(self):
def unlink_failsafe(filename):
try:
os.unlink(fn)
except:
pass
def check_empty_db(filename):
env = {}
assert jsonenv.update_env(fn, env, jsonenv.RAISE)
assert env == {}
env = {"ITEM":"val"}
assert jsonenv.update_env(fn, env, jsonenv.RAISE)
assert env == {"ITEM":"val"}
def check_db(filename, expected=None):
env = {}
assert jsonenv.update_env(fn, env, jsonenv.SKIP)
if expected is not None:
if env != expected:
#print "got:", env
#print "file(%s): <<END" % filename
f = open(fn, "r")
try:
pass
#print f.read()
finally:
f.close()
#print "END"
assert env == expected
d = tempfile.mkdtemp()
try:
#print >> sys.stderr, "Tempdir: %s" % d
fn = os.path.join(d, "empty.db")
try:
check_empty_db(fn)
finally:
unlink_failsafe(fn)
# check non-empty DB will add all values to dict:
env = {"VAR1":"VAL1", "VAR2":"VAL2"}
fn = os.path.join(d, "test.db")
assert jsonenv.export_env(fn, env, jsonenv.RAISE)
try:
check_db(fn, env)
env2 = {}
assert jsonenv.update_env(fn, env2, jsonenv.RAISE)
assert env == env2
finally:
unlink_failsafe(fn)
# check RAISE:
env = {"VAR1":"VAL1", "VAR2":"VAL2", "BAD":object()}
fn = os.path.join(d, "test.db")
try:
try:
jsonenv.export_env(fn, env, jsonenv.RAISE)
assert False
except:
pass
check_empty_db(fn)
finally:
unlink_failsafe(fn)
# check ABORT:
env = {"VAR1":"VAL1", "VAR2":"VAL2", "BAD":object()}
fn = os.path.join(d, "test.db")
try:
assert not jsonenv.export_env(fn, env, jsonenv.ABORT)
check_empty_db(fn)
finally:
unlink_failsafe(fn)
# check SKIP:
env = {"VAR1":"VAL1", "VAR2":"VAL2", "BAD":object()}
fn = os.path.join(d, "test.db")
try:
assert not jsonenv.export_env(fn, env, jsonenv.SKIP)
env2 = {}
assert jsonenv.update_env(fn, env2, jsonenv.RAISE)
env3 = dict(env)
del env3["BAD"]
assert env2 == env3
finally:
unlink_failsafe(fn)
finally:
shutil.rmtree(d)
def testMain(self):
def unlink_failsafe(filename):
try:
os.unlink(fn)
except:
pass
def check_empty_db(filename):
env = {}
assert jsonenv.update_env(fn, env, jsonenv.RAISE)
assert env == {}
env = {"ITEM": "val"}
assert jsonenv.update_env(fn, env, jsonenv.RAISE)
assert env == {"ITEM": "val"}
def check_db(filename, expected=None):
env = {}
assert jsonenv.update_env(fn, env, jsonenv.SKIP)
if expected is not None:
if env != expected:
#print "got:", env
#print "file(%s): <<END" % filename
f = open(fn, "r")
try:
pass
#print f.read()
finally:
f.close()
#print "END"
assert env == expected
d = tempfile.mkdtemp()
try:
#print >> sys.stderr, "Tempdir: %s" % d
fn = os.path.join(d, "empty.db")
try:
check_empty_db(fn)
finally:
unlink_failsafe(fn)
# check non-empty DB will add all values to dict:
env = {"VAR1": "VAL1", "VAR2": "VAL2"}
fn = os.path.join(d, "test.db")
assert jsonenv.export_env(fn, env, jsonenv.RAISE)
try:
check_db(fn, env)
env2 = {}
assert jsonenv.update_env(fn, env2, jsonenv.RAISE)
assert env == env2
finally:
unlink_failsafe(fn)
# check RAISE:
env = {"VAR1": "VAL1", "VAR2": "VAL2", "BAD": object()}
fn = os.path.join(d, "test.db")
try:
try:
jsonenv.export_env(fn, env, jsonenv.RAISE)
assert False
except:
pass
check_empty_db(fn)
finally:
unlink_failsafe(fn)
# check ABORT:
env = {"VAR1": "VAL1", "VAR2": "VAL2", "BAD": object()}
fn = os.path.join(d, "test.db")
try:
assert not jsonenv.export_env(fn, env, jsonenv.ABORT)
check_empty_db(fn)
finally:
unlink_failsafe(fn)
# check SKIP:
env = {"VAR1": "VAL1", "VAR2": "VAL2", "BAD": object()}
fn = os.path.join(d, "test.db")
try:
assert not jsonenv.export_env(fn, env, jsonenv.SKIP)
env2 = {}
assert jsonenv.update_env(fn, env2, jsonenv.RAISE)
env3 = dict(env)
del env3["BAD"]
assert env2 == env3
finally:
unlink_failsafe(fn)
finally:
shutil.rmtree(d)
