def do_env_vars(run_list,action) :
dprint(1,"* Entering do_env_vars")
# make just one list
all_items = run_list[:]
# set or unset each item
for item in all_items :
if action == "set" :
# in the event that the var contains previously-defined vars, evaluate them in the shell first
cmd = "echo " + item.value
stat,out = commands.getstatusoutput(cmd)
dprint(3," * Setting env var " + item.key + " = " + out + " (" + item.value + ")")
os.putenv(item.key,out)
elif action == "unset" :
dprint(3," * Unsetting env var " + item.key + " = " + item.value)
os.unsetenv(item.key)
else :
# unknown action
print "INTERNALERROR: unsupported action %s" % action
sys.exit(10)
# end if
# end for
dprint(1,"* Leaving do_env_vars")
def dump_database(config):
# Create sql file
date = datetime.now().strftime('%w')
filename = '%s-%s.sql' % (config.database, date)
file = os.path.join(config.outputdir, filename)
dumpfile = open(file, 'w')
os.chmod(file, 0600)
# Create command
if config.dbtype == 'mysql':
cmd = ['mysqldump', '-h', config.hostname, '-u', config.username, '-p%s' % config.password, config.database]
elif config.dbtype == 'pgsql':
os.putenv('PGPASSWORD', config.password)
cmd = ['pg_dump', '-h', config.hostname, '-U', config.username, config.database]
else:
sys.exit("Unsupported database type '%s'" % config.dbtype)
# Execute command w/stdout to sql file
proc = subprocess.Popen(cmd, stdout=dumpfile, stderr=subprocess.PIPE)
retcode = proc.wait()
dumpfile.close()
if retcode > 0:
print "Failed to backup database '%s'" % config.database
stderr = proc.stderr.read()
print stderr.strip()
os.remove(file)
else:
compress_file(config, filename)
os.unsetenv('PGPASSWORD')
def testEndpointAndInterface(self):
# Unset DLS_ENPOINT, will try the options
unsetenv("DLS_ENDPOINT")
unsetenv("DBS_CLIENT_CONFIG")
# First, wrong interface selection
cmd = self.path + "/dls-add --skip-location-check -i SOMETHING c1"
st, out = run(cmd)
expected = "Unsupported interface type: SOMETHING\nSupported values: ['DLS_TYPE_LFC', 'DLS_TYPE_MYSQL', 'DLS_TYPE_DBS']"
msg = "Results (%s) are not those expected (%s)" % (out, expected)
self.assertEqual(out, expected, msg)
# Next the DBS URL
cmd = self.path + "/dls-add --skip-location-check %s"%(self.f1)
st, out = run(cmd)
expected = "Error when binding the DLS interface"
expected2 = "Could not set the interface to the DLS server"
msg = "Results (%s) don't contain expected (%s, %s)" % (out, expected, expected2)
contains = (out.find(expected) != -1) and (out.find(expected2) != -1)
self.assert_(contains, msg)
# Now correct endpoint but incorrect type
cmd = self.path + "/dls-add --skip-location-check -e %s%s -i DLS_TYPE_DLI %s CliTest-se1" % (self.host, self.testdir, self.f1)
st, out = run(cmd)
expected = "Unsupported interface type: DLS_TYPE_DLI"
msg = "Results (%s) don't contain expected (%s)" % (out, expected)
contains = (out.find(expected) != -1)
self.assert_(contains, msg)
# With everything right, the thing should work
putenv("DBS_CLIENT_CONFIG", self.dbscfg)
cmd = self.path + "/dls-add --skip-location-check -e %s%s %s CliTest-se1"%(self.host, self.testdir, self.f1)
st, out = run(cmd)
msg = "Error in %s: %s" % (cmd, out)
self.assertEqual(st, 0, msg)
cmd = self.path + "/dls-add --skip-location-check -e %s%s -i DLS_TYPE_DBS %s CliTest-se2" % (self.host, self.testdir, self.f2)
st, out = run(cmd)
msg = "Error in %s: %s" % (cmd, out)
self.assertEqual(st, 0, msg)
# No update yet
# cmd = self.path + "/dls-update -e %s%s c1 filesize=100" % (self.host, self.testdir)
# st, out = run(cmd)
# msg = "Error in dls-update -e %s%s c1 filesize=100: %s" % (self.host, self.testdir, out)
# self.assertEqual(st, 0, msg)
cmd = self.path + "/dls-get-se -e %s%s %s"%(self.host, self.testdir, self.f1)
st, out = run(cmd)
msg = "Error in dls-get-se -e %s%s %s: %s" % (self.host, self.testdir, self.f1, out)
self.assertEqual(st, 0, msg)
cmd = self.path + "/dls-get-fileblock -e %s%s CliTest-NOTHING" % (self.host, self.testdir)
st, out = run(cmd)
msg = "Error in dls-get-fileblock -e %s%s CliTest-NOTHING: %s" % (self.host, self.testdir, out)
self.assertEqual(st, 0, msg)
cmd = self.path + "/dls-delete -e %s%s -a %s" % (self.host, self.testdir,self.f1)
st, out = run(cmd)
msg = "Error in dls-delete -e %s%s -a %s: %s" % (self.host,self.testdir,self.f1,out)
self.assertEqual(st, 0, msg)
def execute(self, args=()):
entry_point = self.entry()
with mutable_sys():
sys.path, sys.path_importer_cache = self.minimum_path()
self._env.activate()
if 'PEX_COVERAGE' in os.environ:
PEX.start_coverage()
TRACER.log('PYTHONPATH contains:')
for element in sys.path:
TRACER.log(' %c %s' % (' ' if os.path.exists(element) else '*', element))
TRACER.log(' * - paths that do not exist or will be imported via zipimport')
force_interpreter = 'PEX_INTERPRETER' in os.environ
if entry_point and not force_interpreter:
self.execute_entry(entry_point, args)
else:
os.unsetenv('PEX_INTERPRETER')
TRACER.log('%s, dropping into interpreter' % (
'PEX_INTERPRETER specified' if force_interpreter else 'No entry point specified'))
if sys.argv[1:]:
try:
with open(sys.argv[1]) as fp:
ast = compile(fp.read(), fp.name, 'exec', flags=0, dont_inherit=1)
except IOError as e:
print("Could not open %s in the environment [%s]: %s" % (sys.argv[1], sys.argv[0], e))
sys.exit(1)
sys.argv = sys.argv[1:]
old_name = globals()['__name__']
try:
globals()['__name__'] = '__main__'
exec_function(ast, globals())
finally:
globals()['__name__'] = old_name
else:
import code
code.interact()
def set_testing_environment(val):
if val:
environ['TETHYS_TESTING_IN_PROGRESS'] = 'true'
else:
environ['TETHYS_TESTING_IN_PROGRESS'] = ''
del environ['TETHYS_TESTING_IN_PROGRESS']
unsetenv('TETHYS_TESTING_IN_PROGRESS')
def run(self, binary=None, interpreter_args=[], args=[], extra_deps=[], with_chroot=False):
path = OrderedSet(self._path)
Launcher._setup_eggs(path, extra_deps)
if os.getenv('PANTS_NO_SITE'):
cmdline = [sys.executable, '-S'] + interpreter_args
else:
cmdline = [sys.executable] + interpreter_args
bin = binary
if not bin: bin = self._binary
if bin:
cmdline.append(bin)
if args:
cmdline.extend(args)
cwd = os.getcwd()
oldenv = os.getenv('PYTHONPATH')
os.putenv('PYTHONPATH', ':'.join(path))
if with_chroot:
os.chdir(self._dir)
print 'Executing PYTHONPATH=%s %s' % (
':'.join(path), ' '.join(cmdline))
po = subprocess.Popen(cmdline)
rv = po.wait()
if with_chroot:
os.chdir(cwd)
if oldenv:
os.putenv('PYTHONPATH', oldenv)
else:
os.unsetenv('PYTHONPATH')
return rv
def align(sequence, auto=True, cpu=4, outdir=OUTDIR, indir=INDIR):
"""function that will parallelize protein alignment
If alignement has less than 250 sequence, 10'000 iteration are done"""
outfile = ".".join(os.path.basename(sequence).split(".")[:-1])+".aln"
n_seq = subprocess.check_output(["grep", "-c", ">", sequence])
if int(n_seq) < 250:
N_iter = 10000
else:
N_iter = 1000
os.unsetenv("MAFFT_BINARIES") # Needed if T-Coffee installed.
with open(os.path.join(indir, outfile), "w") as outf:
if auto:
mafft_cmd = ["/usr/local/bin/mafft",
"--thread", str(cpu),
"--auto",
sequence]
else:
mafft_cmd = ["/usr/local/bin/mafft",
"--thread", str(cpu),
"--maxiterate", str(N_iter),
"--localpair",
sequence]
returncode = subprocess.call(mafft_cmd, stdout=outf)
if returncode!=0:
raise "Error, returncode = {}".format(returncode)
def tearDown(self):
self.removeRecursive(self.dir)
self.removeRecursive(self.cdir)
if self.cvsroot:
os.environ["CVSROOT"] = self.cvsroot
else:
os.unsetenv("CVSROOT")
def testCacheDir(self):
# No cache dir set, None is returned.
cache_dir = resolver.tfhub_cache_dir()
self.assertEqual(cache_dir, None)
# Use temp dir.
cache_dir = resolver.tfhub_cache_dir(use_temp=True)
self.assertEquals(cache_dir,
os.path.join(tempfile.gettempdir(), "tfhub_modules"))
# Use override
cache_dir = resolver.tfhub_cache_dir(default_cache_dir="/d", use_temp=True)
self.assertEqual("/d", cache_dir)
# Use a flag
FLAGS.tfhub_cache_dir = "/e"
cache_dir = resolver.tfhub_cache_dir(default_cache_dir="/d", use_temp=True)
self.assertEqual("/e", cache_dir)
FLAGS.tfhub_cache_dir = ""
# Use env variable
os.environ[resolver._TFHUB_CACHE_DIR] = "/f"
cache_dir = resolver.tfhub_cache_dir(default_cache_dir="/d", use_temp=True)
self.assertEqual("/f", cache_dir)
FLAGS.tfhub_cache_dir = "/e"
cache_dir = resolver.tfhub_cache_dir(default_cache_dir="/d", use_temp=True)
self.assertEqual("/f", cache_dir)
FLAGS.tfhub_cache_dir = ""
os.unsetenv(resolver._TFHUB_CACHE_DIR)
def inve(env, command, args):
"""Enter a virtual environment"""
path = workon_home() / env
if not path.exists():
sys.exit("Environment '{}' does not exist.".format(env))
if not command:
command = 'powershell' if windows else os.environ['SHELL']
with chdir(project_dir(path) or Path.cwd()):
with temp_environ():
os.environ['VIRTUAL_ENV'] = str(path)
os.environ['PATH'] = os.pathsep.join(
[str(path / bin), os.environ['PATH']])
os.unsetenv('PYTHONHOME')
os.unsetenv('__PYVENV_LAUNCHER__')
try:
sys.exit(call([command] + list(args), shell=windows))
# need to have shell=True on windows, otherwise the PYTHONPATH
# won't inherit the PATH
except OSError as e:
if e.errno == 2:
click.echo('Unable to find {}'.format(command))
else:
raise
def setUp(self):
self.dir = tempfile.mkdtemp(suffix=".bigitr")
self.cdir = tempfile.mkdtemp(suffix=".bigitr")
self.fdir = "%s/repo/@{trunk}/Loc" % self.cdir
os.makedirs(self.fdir)
if "CVSROOT" in os.environ:
self.cvsroot = os.environ["CVSROOT"]
else:
self.cvsroot = None
os.unsetenv("CVSROOT")
with mock.patch("bigitr.log.Log") as mocklog:
appConfig = StringIO(
"[global]\n" "logdir = /logs\n" "gitdir = %s\n" "[export]\n" "cvsdir = %s\n" % (self.dir, self.cdir)
)
repConfig = StringIO(
"[GLOBAL]\n"
"[repo]\n"
"cvsroot = asdf\n"
"cvspath = Some/Loc\n"
"prehook.cvs = precommand arg\n"
"posthook.cvs = postcommand arg\n"
"prehook.cvs.@{trunk} = precommand trunk\n"
"posthook.cvs.@{trunk} = postcommand trunk\n"
"\n"
)
self.ctx = context.Context(appConfig, repConfig)
self.cvs = cvs.CVS(self.ctx, "repo", "@{trunk}")
self.mocklog = mocklog()
def load_environment(old_env, new_env):
"""Load up new copy of environment from a dict."""
u.verbose(2, "load_environment invoked")
# Find new vars, modified vars, deleted vars
added_vars = {}
modified_vars = {}
deleted_vars = {}
for v in new_env:
if v not in old_env:
added_vars[v] = 1
else:
if new_env[v] != old_env[v]:
modified_vars[v] = 1
for v in old_env:
if v not in new_env:
deleted_vars[v] = 1
u.verbose(2, "deleting vars: %s" % " ".join(sorted(deleted_vars.keys())))
for v in deleted_vars:
os.unsetenv(v)
os.environ.pop(v, None)
u.verbose(2, "adding vars: %s" % " ".join(sorted(added_vars.keys())))
for v in added_vars:
os.putenv(v, new_env[v])
os.environ[v] = new_env[v]
u.verbose(2, "modifying vars: %s" % " ".join(sorted(modified_vars.keys())))
for v in modified_vars:
os.putenv(v, new_env[v])
os.environ[v] = new_env[v]
def __init__(self, hosts = ['http://localhost/server-status?auto'], mode = ApacheTopWidescreen, interval = 1):
self.hosts = hosts
self.interval = interval
self.modes = [ApacheTopTabular, ApacheTopWidescreen]
self.itermodes = iter(self.modes)
# fix curses / readline bug during window resize
os.unsetenv('LINES')
os.unsetenv('COLUMNS')
self.scr = curses.initscr()
self.scr.nodelay(1)
self.mode = mode(self.scr)
curses.start_color()
curses.curs_set(0)
curses.init_pair(1, curses.COLOR_BLUE, curses.COLOR_BLACK)
curses.init_pair(2, curses.COLOR_GREEN, curses.COLOR_BLACK)
curses.init_pair(3, curses.COLOR_RED, curses.COLOR_BLACK)
curses.init_pair(4, curses.COLOR_YELLOW, curses.COLOR_BLACK)
curses.init_pair(5, curses.COLOR_WHITE, curses.COLOR_BLUE)
signal(SIGINT, self.cleanup)
self.exit = False
def tearDown(self):
os.close(self.fd)
os.remove(self.cf)
if self.fakehome:
os.environ['FAKEHOME'] = self.fakehome
else:
os.unsetenv('FAKEHOME')
def cmd_man(msg):
args = ["man"]
num = None
if len(msg.args) == 1:
args.append(msg.args[0])
elif len(msg.args) >= 2:
try:
num = int(msg.args[0])
args.append("%d" % num)
args.append(msg.args[1])
except ValueError:
args.append(msg.args[0])
os.unsetenv("LANG")
res = Response(channel=msg.channel)
with subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as proc:
for line in proc.stdout.read().split(b"\n"):
(line, n) = RGXP_s.subn(b'', line)
res.append_message(line.decode())
if len(res.messages) <= 0:
if num is not None:
res.append_message("There is no entry %s in section %d." %
(msg.args[0], num))
else:
res.append_message("There is no man page for %s." % msg.args[0])
return res
def empty_environment():
"""
Remove all variables from the environment.
"""
for s in os.environ.keys():
os.unsetenv(s)
del os.environ[s]
def build():
# Prepare files
os.chdir('data')
if 'debug' in sys.argv:
subprocess.call(go_bindata_debug)
else:
subprocess.call(go_bindata)
os.chdir('..')
if 'release' in sys.argv:
for os1 in OSS:
for arch in ARCHS:
os.putenv('GOOS', os1)
os.putenv('GOARCH', arch)
filename = '{}_{}_{}'.format(PROJECT, os1, arch)
if os1 == 'windows':
filename += '.exe'
build_cmd = 'go build -o {}'.format(filename).split(' ')
exit_code = subprocess.call(build_cmd)
check_error(exit_code, os1, arch)
os.unsetenv('GOARCH')
os.unsetenv('GOOS')
exit_code = subprocess.call('go build'.split(' '))
check_error(exit_code)
def _create_database(self, key):
"""Create database if it doesn't exist."""
conn_string = self.test_databases[key]
conn_pieces = urlparse.urlparse(conn_string)
if conn_string.startswith("mysql"):
(user, password, database, host) = get_mysql_connection_info(conn_pieces)
sql = "create database if not exists %s;" % database
cmd = 'mysql -u "%(user)s" %(password)s -h %(host)s ' '-e "%(sql)s"' % {
"user": user,
"password": password,
"host": host,
"sql": sql,
}
self.execute_cmd(cmd)
elif conn_string.startswith("postgresql"):
(user, password, database, host) = get_pgsql_connection_info(conn_pieces)
os.environ["PGPASSWORD"] = password
os.environ["PGUSER"] = user
sqlcmd = "psql -w -U %(user)s -h %(host)s -c" " '%(sql)s' -d template1"
sql = ("create database if not exists %s;") % database
createtable = sqlcmd % {"user": user, "host": host, "sql": sql}
# 0 means databases is created
# 256 means it already exists (which is fine)
# otherwise raise an error
out, err = processutils.trycmd(createtable, shell=True, check_exit_code=[0, 256], discard_warnings=True)
output = out or err
if err != "":
self.fail("Failed to run: %s\n%s" % (createtable, output))
os.unsetenv("PGPASSWORD")
os.unsetenv("PGUSER")
def build_bundle(args):
version = args.version
temp_dir = tempfile.mkdtemp()
start_dir = os.getcwd()
try:
os.putenv('CONDA_ENVS_PATH', temp_dir)
# 1. Install OpenMDAO to a temporary conda environment
# 2. Grab all packages
# 3. Make tar file
create_env(
'openmdao-bundle',
['openmdao=={version}'.format(version=version)],
channel='http://conda.binstar.org/openmdao',
yes=True
)
os.chdir('{envs_path}/.pkgs'.format(envs_path=temp_dir))
pkgs = glob.glob('*.tar.bz2')
out = tarfile.open('openmdao.tar', mode='w')
with tarfile.open('openmdao.tar', mode='w') as tar:
for pkg in pkgs:
tar.add(pkg, recursive=False)
shutil.move(
'openmdao.tar',
'{start_dir}/openmdao.tar'.format(start_dir=start_dir)
)
finally:
os.chdir(start_dir)
os.unsetenv('CONDA_ENVS_PATH')
shutil.rmtree(temp_dir)
def do_test(self, buildscript):
buildscript.set_action('Testing', self)
if not buildscript.config.noxvfb:
# start Xvfb
old_display = os.environ.get('DISPLAY')
old_xauth = os.environ.get('XAUTHORITY')
xvfb_pid = self._start_xvfb(buildscript.config.xvfbargs)
if xvfb_pid == -1:
raise BuildStateError('Unable to start Xvfb')
# either do_ldtp_test or do_dogtail_test
method = getattr(self, 'do_' + self.test_type + '_test')
try:
method(buildscript)
finally:
if not buildscript.config.noxvfb:
# kill Xvfb if it has been started
self._stop_xvfb(xvfb_pid)
if old_display:
os.environ['DISPLAY'] = old_display
else:
os.unsetenv('DISPLAY')
if old_xauth:
os.environ['XAUTHORITY'] = old_xauth
else:
os.unsetenv('XAUTHORITY')
def load_modules(self,*k,**kw):
module_string = ''
try:
for module in kw['modules']:
module_string += module+' '
except KeyError:
self.fatal('You must give modules to function check_modules like this: check_module(modules=[a,b,c])')
#self.start_msg('Loading modules')
p = subprocess.Popen('source /usr/local/Modules/current/init/bash && module load '+module_string+' && export -p', shell=True, stdout=subprocess.PIPE)
p.wait()
if p.returncode == 0:
for key in os.environ.iterkeys():
os.unsetenv(key)
for line in p.stdout:
m = re.search('(\w+)=(".+")$', line)
if (m):
os.putenv(m.group(1), m.group(2))
#self.end_msg(module_string)
else:
self.fatal('Loading modules did not work')
def initialize(datadumpdirectory=None, indexgit=None, customindexfile=None):
"""Initializes some variables, as required, including setting up defaults"""
Logger().log(Logger.info, "Initializing systems...")
if datadumpdirectory is not None:
Globals.Globals.setdatadirectory(datadumpdirectory)
else:
Globals.Globals.setdatadirectory("./cccp-index-test")
if indexgit is not None:
Globals.Globals.indexGit = indexgit
if customindexfile is not None:
if os.path.exists(customindexfile):
Globals.Globals.customIndexFile = customindexfile
else:
Logger().log(Logger.error, "Non existant custom index specified, falling back to index git.")
Globals.Globals.oldenviron = dict(os.environ)
os.unsetenv("GIT_ASKPASS")
os.unsetenv("SSH_ASKPASS")
StaticHandler._setupFS()
return
def cmd_pydoc(args):
import init
init.init_pydoc()
import pydoc, os
try:
oldargv = sys.argv
except AttributeError:
oldargv = None
oldpath = sys.path
oldterm = os.getenv("TERM")
try:
sys.argv = args
sys.path = [''] + oldpath
os.putenv("TERM", "dumb")
pydoc.cli()
finally:
if oldargv is None:
del sys.argv
else:
sys.argv = oldargv
sys.path = oldpath
if oldterm is None:
os.unsetenv("TERM")
else:
os.putenv("TERM", oldterm)
def clear_environment():
language = os.getenv("LANG")
for key in os.environ.keys():
os.unsetenv(key)
os.putenv("DISPLAY", ":0")
os.putenv("LANG", language)
os.putenv("PATH", "/bin:/usr/bin")
def execute(self, args=()):
entry_point = self.entry()
with mutable_sys():
sys.path, sys.path_importer_cache = self.minimum_path()
self._env.activate()
if 'PEX_COVERAGE' in os.environ:
PEX.start_coverage()
TRACER.log('PYTHONPATH now %s' % ':'.join(sys.path))
force_interpreter = 'PEX_INTERPRETER' in os.environ
if entry_point and not force_interpreter:
self.execute_entry(entry_point, args)
else:
os.unsetenv('PEX_INTERPRETER')
TRACER.log('%s, dropping into interpreter' % (
'PEX_INTERPRETER specified' if force_interpreter else 'No entry point specified.'))
if sys.argv[1:]:
try:
with open(sys.argv[1]) as fp:
ast = compile(fp.read(), fp.name, 'exec')
except IOError as e:
print("Could not open %s in the environment [%s]: %s" % (sys.argv[1], sys.argv[0], e))
sys.exit(1)
sys.argv = sys.argv[1:]
old_name = globals()['__name__']
try:
globals()['__name__'] = '__main__'
Compatibility.exec_function(ast, globals())
finally:
globals()['__name__'] = old_name
else:
import code
code.interact()
def filter_environment(good_vars):
"""
Create a pristine environment for bitbake. This will remove variables that
are not known and may influence the build in a negative way.
"""
removed_vars = {}
for key in os.environ.keys():
if key in good_vars:
continue
removed_vars[key] = os.environ[key]
os.unsetenv(key)
del os.environ[key]
# If we spawn a python process, we need to have a UTF-8 locale, else python's file
# access methods will use ascii. You can't change that mode once the interpreter is
# started so we have to ensure a locale is set. Ideally we'd use C.UTF-8 but not all
# distros support that and we need to set something.
os.environ["LC_ALL"] = "en_US.UTF-8"
if removed_vars:
logger.debug(1, "Removed the following variables from the environment: %s", ", ".join(removed_vars.keys()))
return removed_vars
def setup():
subprocess.check_call([
sys.executable, 'build.py', '--dest', '.test/longrun',
'--kit', 'platform', '--disable-response-cache'])
os.putenv('MALLOC_TRIM_THRESHOLD_', '0')
server = integration_tests.common.start_server(
'integration_tests/dump.py', path='.test/longrun', sanitize=False)
os.unsetenv('MALLOC_TRIM_THRESHOLD_')
os.makedirs('.collector', exist_ok=True)
os.putenv('COLLECTOR_FILE', '.collector/{}.json'.format(server.pid))
collector = subprocess.Popen([
sys.executable, 'misc/collector.py', str(server.pid)])
os.unsetenv('COLLECTOR_FILE')
def cleanup(*args):
try:
server.terminate()
assert server.wait() == 0
finally:
atexit.unregister(cleanup)
atexit.register(cleanup)
signal.signal(signal.SIGINT, cleanup)
def test_counters(self):
from pypy.rpython.lltypesystem import lltype
from pypy.rpython.lltypesystem.lloperation import llop
def entry_point(argv):
llop.instrument_count(lltype.Void, 'test', 2)
llop.instrument_count(lltype.Void, 'test', 1)
llop.instrument_count(lltype.Void, 'test', 1)
llop.instrument_count(lltype.Void, 'test', 2)
llop.instrument_count(lltype.Void, 'test', 1)
return 0
t = TranslationContext(self.config)
t.config.translation.instrument = True
t.buildannotator().build_types(entry_point, [s_list_of_strings])
t.buildrtyper().specialize()
cbuilder = CStandaloneBuilder(t, entry_point, config=t.config) # xxx
cbuilder.generate_source()
cbuilder.compile()
counters_fname = udir.join("_counters_")
os.putenv('_INSTRUMENT_COUNTERS', str(counters_fname))
try:
data = cbuilder.cmdexec()
finally:
os.unsetenv('_INSTRUMENT_COUNTERS')
f = counters_fname.open('rb')
counters_data = f.read()
f.close()
import struct
counters = struct.unpack("LLL", counters_data)
assert counters == (0,3,2)
def run(self):
# base install
_install.run(self)
# custom install of backend
log.info('Now installing cppyy-cling into cppyy_backend')
builddir = get_builddir()
if not os.path.exists(builddir):
raise DistutilsSetupError('Failed to find build dir!')
# use $MAKE to install if it is defined
env_make = os.getenv("MAKE")
if not env_make:
install_cmd = 'cmake'
install_args = ['--build', '.', '--config', get_build_type(), '--target', 'install']
else:
install_args = env_make.split()
install_cmd, install_args = install_args[0], install_args[1:]+['install']
prefix = get_prefix()
log.info('Now creating installation under %s ...', prefix)
if env_make: os.unsetenv("MAKE")
if subprocess.call([install_cmd] + install_args, cwd=builddir) != 0:
raise DistutilsSetupError('Failed to install cppyy-cling')
if env_make: os.putenv("MAKE", env_make)
if 'linux' in sys.platform:
# remove allDict.cxx.pch as it's not portable (rebuild on first run, see cppyy)
log.info('removing allDict.cxx.pch')
os.remove(os.path.join(get_prefix(), 'etc', 'allDict.cxx.pch'))
# for manylinux, reset the default cxxversion to 17 if no user override
if not 'STDCXX' in os.environ and is_manylinux():
log.info('updating root-config to C++17 for manylinux')
inp = os.path.join(get_prefix(), 'bin', 'root-config')
outp = inp+'.new'
outfile = open(outp, 'w')
for line in open(inp).readlines():
if 'cxxversion=' == line[:11]:
line = 'cxxversion=cxx17\n'
outfile.write(line)
outfile.close()
os.rename(outp, inp)
os.chmod(inp, stat.S_IMODE(os.lstat(inp).st_mode) | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
log.info('updating allCppflags.txt to C++17 for manylinux')
inp = os.path.join(get_prefix(), 'etc', 'dictpch', 'allCppflags.txt')
outp = inp+'.new'
outfile = open(outp, 'w')
for line in open(inp).readlines():
if '-std=' == line[:5]:
line = '-std=c++1z\n'
outfile.write(line)
outfile.close()
os.rename(outp, inp)
install_path = self._get_install_path()
log.info('Copying installation to: %s ...', install_path)
self.copy_tree(os.path.join(get_prefix(), os.path.pardir), install_path)
log.info('Install finished')
def read_gateway( config_dir, gateway_name ):
"""
Read a gateway
Return a dict with the gateway attributes on success
Raise on error
"""
env = make_env()
del env['SYNDICATE_DEBUG']
SYNDICATE_DEBUG = False
if os.environ.has_key('SYNDICATE_DEBUG'):
SYNDICATE_DEBUG = os.environ['SYNDICATE_DEBUG']
os.unsetenv('SYNDICATE_DEBUG')
print 'SYNDICATE_DEBUG = {}'.format(os.environ.get('SYNDICATE_DEBUG', '(unset)'))
exitcode, out = run(testconf.SYNDICATE_TOOL,
'-c', os.path.join(config_dir, 'syndicate.conf'),
'read_gateway',
gateway_name, env=env)
if SYNDICATE_DEBUG:
os.environ['SYNDICATE_DEBUG'] = SYNDICATE_DEBUG
if exitcode != 0:
print >> sys.stderr, out
raise Exception("%s exited %s" % (testconf.SYNDICATE_TOOL, exitcode))
try:
ret = json.loads(out)
except:
raise Exception("Invalid output:\n%s\n" % out)
return ret
def main(sdk_path, test_pattern):
logging.disable(logging.WARNING)
start_time = time.time()
os.environ['IS_TBA_TEST'] = "true"
# Fix django template loaders being messed up
import django.conf.global_settings
django.conf.global_settings.SECRET_KEY = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
os.environ.setdefault('DJANGO_SETTINGS_MODULE',
'django.conf.global_settings')
sys.path.insert(0, sdk_path)
import dev_appserver
dev_appserver.fix_sys_path()
# Set up custom django template filters
from google.appengine.ext.webapp import template
template.register_template_library('common.my_filters')
suites = unittest2.loader.TestLoader().discover("tests", test_pattern)
fail = False
total_tests_run = 0
if MULTITHREAD:
proc_lock = multiprocessing.Lock()
fail_count = multiprocessing.Value('i', 0)
total_run = multiprocessing.Value('i', 0)
pool = multiprocessing.Pool(MAX_JOBS,
initializer=proc_init,
initargs=(
proc_lock,
fail_count,
total_run,
))
pool.map(run_suite, suites)
pool.close()
pool.join()
fail = fail_count.value > 0
total_tests_run = total_run.value
else:
result_queue = multiprocessing.Queue()
for suite in suites:
sio = StringIO.StringIO()
testresult = unittest2.TextTestRunner(sio, verbosity=2).run(suite)
result_queue.put((testresult.testsRun, testresult.wasSuccessful()))
print '-----------------------'
print sio.getvalue().encode('utf-8')
while not result_queue.empty():
tests_run, was_successful = result_queue.get()
total_tests_run += tests_run
if not was_successful:
fail = True
os.unsetenv('IS_TBA_TEST')
print "================================"
print "Completed {} tests in: {} seconds".format(total_tests_run,
time.time() - start_time)
if fail:
print "TESTS FAILED!"
else:
print "TESTS PASSED!"
print "================================"
if fail:
sys.exit(1)
else:
sys.exit(0)
def unsetenvproxy():
"""Unset HTTP Proxy environment variables that might interfere"""
for proxyvar in [ 'http_proxy', 'HTTP_PROXY' ]:
os.unsetenv(proxyvar)
return
bufsize=BUFFER_SIZE,
stdin=subprocess.PIPE,
stdout=childStdout,
stderr=childStderr,
close_fds=True)
return proc
#randomize port and dir, so multiple processes works
ckptDir = "dmtcp-autotest-%d" % randint(100000000, 999999999)
os.mkdir(ckptDir)
os.environ['DMTCP_COORD_HOST'] = "localhost"
os.environ['DMTCP_COORD_PORT'] = str(randint(2000, 10000))
os.environ['DMTCP_CHECKPOINT_DIR'] = os.path.abspath(ckptDir)
#Use default SIGCKPT for test suite.
os.unsetenv('DMTCP_SIGCKPT')
os.unsetenv('MTCP_SIGCKPT')
#No gzip by default. (Isolate gzip failures from other test failures.)
#But note that dmtcp3, frisbee and gzip tests below still use gzip.
if not args.verbose:
os.environ['JALIB_STDERR_PATH'] = os.devnull
if args.verbose:
print "coordinator port: " + os.environ['DMTCP_COORD_PORT']
# We'll copy ckptdir to DMTCP_TMPDIR in case of error.
def dmtcp_tmpdir():
tmpdir = os.getenv('DMTCP_TMPDIR') or os.getenv('TMPDIR') or '/tmp'
return tmpdir + '/dmtcp-' + os.environ['USER'] + '@' + socket.gethostname()
def setup_kwargs(self, kwargs):
if kwargs["binary"] is None:
if kwargs["browser_channel"] is None:
kwargs["browser_channel"] = "nightly"
logger.info(
"No browser channel specified. Running nightly instead.")
binary = self.browser.find_binary(self.venv.path,
kwargs["browser_channel"])
if binary is None:
raise WptrunError("""Firefox binary not found on $PATH.
Install Firefox or use --binary to set the binary path""")
kwargs["binary"] = binary
if kwargs["certutil_binary"] is None and kwargs["ssl_type"] != "none":
certutil = self.browser.find_certutil()
if certutil is None:
# Can't download this for now because it's missing the libnss3 library
logger.info(
"""Can't find certutil, certificates will not be checked.
Consider installing certutil via your OS package manager or directly.""")
else:
logger.info("Using certutil %s" % certutil)
kwargs["certutil_binary"] = certutil
if kwargs["webdriver_binary"] is None and "wdspec" in kwargs[
"test_types"]:
webdriver_binary = self.browser.find_webdriver()
if webdriver_binary is None:
install = self.prompt_install("geckodriver")
if install:
logger.info("Downloading geckodriver")
webdriver_binary = self.browser.install_webdriver(
dest=self.venv.bin_path)
else:
logger.info("Using webdriver binary %s" % webdriver_binary)
if webdriver_binary:
kwargs["webdriver_binary"] = webdriver_binary
else:
logger.info(
"Unable to find or install geckodriver, skipping wdspec tests"
)
kwargs["test_types"].remove("wdspec")
if kwargs["prefs_root"] is None:
prefs_root = self.browser.install_prefs(
kwargs["binary"],
self.venv.path,
channel=kwargs["browser_channel"])
kwargs["prefs_root"] = prefs_root
if kwargs["headless"] is None:
kwargs["headless"] = True
logger.info(
"Running in headless mode, pass --no-headless to disable")
# Turn off Firefox WebRTC ICE logging on WPT (turned on by mozrunner)
os.unsetenv('R_LOG_LEVEL')
os.unsetenv('R_LOG_DESTINATION')
os.unsetenv('R_LOG_VERBOSE')
# Allow WebRTC tests to call getUserMedia.
kwargs["extra_prefs"].append("media.navigator.streams.fake=true")
def __init__(self, callbacks, types = [InputBox, TagList]):
CommandHandler.__init__(self)
self.plugin_class = ScreenPlugin
self.update_plugin_lookups()
self.callbacks = callbacks
self.layout = "default"
self.window_types = types
self.stdscr = curses.initscr()
if self.curses_setup() < 0:
return -1
self.pseudo_input_box = curses.newpad(1,1)
self.pseudo_input_box.keypad(1)
self.pseudo_input_box.nodelay(1)
self.input_lock = Lock()
set_redisplay_callback(self.readline_redisplay)
set_getc(self.readline_getc)
# See Python bug 2675, readline + curses
os.unsetenv('LINES')
os.unsetenv('COLUMNS')
self.floats = []
self.tiles = []
self.windows = []
self.subwindows()
args = {
"color_name" : ("[color name] Either a pair number (0-255, >8 ignored on 8 color terminals), a default fore/background (deffg, defbg), or an arbitrary name to be used in themes (unread, pending, etc.)", self.type_color_name),
"fg-color" : ("[fg-color] Foreground color", self.type_color),
"bg-color" : ("[bg-color] Background color (optional)\n\nNamed colors: white black red yellow green blue magenta pink\nNumeric colors: 1-256", self.type_color),
"style" : ("[style] Curses style (normal, bold, dim, reverse, standout, underline)", self.type_style),
}
cmds = {
"color": (self.cmd_color, ["color_name", "fg-color", "bg-color"],
"""Change the color palette.
Most like you want to use this to change a color used in the theme. For example,
:color unread green
Will change the color of unread items to green, with the default background. The list of names used in the default theme are:
unread
read
selected
marked
pending
error
reader_quote
reader_link
reader_image_link
reader_italics
enum_hints
You can also change the defaults
:color deffg blue
:color defbg white
Which will be used anywhere a color pair doesn't have an explicit foreground/background.
Lastly you can change the color pairs themselves. This isn't recommended, they're initialized so each available color is available with the default background. If you change these pairs, the named colors above may not make any sense (i.e. green really turns on the color pair set aside for green, so if you change that pair to actually be yellow, don't expect this command to figure it out).
:color 1 white red
Arguments:"""
),
"style": (self.cmd_style, ["color_name", "style"],
"""Change the curses style of a named color.
For example,
:style selected underline
The names used in the default theme are:
unread
read
selected
marked
pending
error
reader_quote
reader_link
reader_image_link
reader_italics
enum_hints
Changing other colors (numeric pairs or deffg/defbg) will have no effect as these are separate from the built in curses color system.
Arguments:"""),
}
register_arg_types(self, args)
register_commands(self, cmds, "Theme")
on_hook("curses_opt_change", self.screen_opt_change)
def run(test, params, env):
"""
Convert specific xen guest
"""
for v in list(params.values()):
if "V2V_EXAMPLE" in v:
test.cancel("Please set real value for %s" % v)
if utils_v2v.V2V_EXEC is None:
test.cancel('Missing command: virt-v2v')
vm_name = params.get('main_vm')
new_vm_name = params.get('new_vm_name')
xen_host = params.get('xen_hostname')
xen_host_user = params.get('xen_host_user', 'root')
xen_host_passwd = params.get('xen_host_passwd', 'redhat')
output_mode = params.get('output_mode')
v2v_timeout = int(params.get('v2v_timeout', 1200))
status_error = 'yes' == params.get('status_error', 'no')
skip_vm_check = params.get('skip_vm_check', 'no')
skip_reason = params.get('skip_reason')
pool_name = params.get('pool_name', 'v2v_test')
pool_type = params.get('pool_type', 'dir')
pool_target = params.get('pool_target_path', 'v2v_pool')
pvt = libvirt.PoolVolumeTest(test, params)
address_cache = env.get('address_cache')
checkpoint = params.get('checkpoint', '')
bk_list = ['vnc_autoport', 'vnc_encrypt', 'vnc_encrypt_warning']
error_list = []
# For construct rhv-upload option in v2v cmd
output_method = params.get("output_method")
rhv_upload_opts = params.get("rhv_upload_opts")
storage_name = params.get('storage_name')
# for get ca.crt file from ovirt engine
rhv_passwd = params.get("rhv_upload_passwd")
rhv_passwd_file = params.get("rhv_upload_passwd_file")
ovirt_engine_passwd = params.get("ovirt_engine_password")
ovirt_hostname = params.get("ovirt_engine_url").split(
'/')[2] if params.get("ovirt_engine_url") else None
ovirt_ca_file_path = params.get("ovirt_ca_file_path")
local_ca_file_path = params.get("local_ca_file_path")
virsh_instance = None
def log_fail(msg):
"""
Log error and update error list
"""
logging.error(msg)
error_list.append(msg)
def set_graphics(virsh_instance, param):
"""
Set graphics attributes of vm xml
"""
vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(
vm_name, virsh_instance=virsh_instance)
graphic = vmxml.xmltreefile.find('devices').find('graphics')
for key in param:
logging.debug('Set %s=\'%s\'' % (key, param[key]))
graphic.set(key, param[key])
vmxml.sync(virsh_instance=virsh_instance)
def check_grub_file(vmcheck, check):
"""
Check grub file content
"""
logging.info('Checking grub file')
grub_file = utils_misc.get_bootloader_cfg(session=vmcheck.session)
if not grub_file:
test.error('Not found grub file')
content = vmcheck.session.cmd('cat %s' % grub_file)
if check == 'console_xvc0':
if 'console=xvc0' in content:
log_fail('"console=xvc0" still exists')
def check_kernel(vmcheck):
"""
Check content of /etc/sysconfig/kernel
"""
logging.info('Checking /etc/sysconfig/kernel file')
content = vmcheck.session.cmd('cat /etc/sysconfig/kernel')
logging.debug(content)
if 'DEFAULTKERNEL=kernel' not in content:
log_fail('Not find "DEFAULTKERNEL=kernel"')
elif 'DEFAULTKERNEL=kernel-xen' in content:
log_fail('DEFAULTKERNEL is "kernel-xen"')
def check_sound_card(vmcheck, check):
"""
Check sound status of vm from xml
"""
xml = virsh.dumpxml(vm_name,
session_id=vmcheck.virsh_session_id).stdout
logging.debug(xml)
if check == 'sound' and '<sound model' in xml:
log_fail('Sound card should be removed')
if check == 'pcspk' and output_mode == 'libvirt' and "<sound model='pcspk'" not in xml:
log_fail('Sound card should be "pcspk"')
def check_rhsrvany_md5(vmcheck):
"""
Check if MD5 and SHA1 of rhsrvany.exe are correct
"""
logging.info('Check md5 and sha1 of rhsrvany.exe')
val_md5, val_sha1 = params.get('val_md5'), params.get('val_sha1')
logging.info('Expect MD5=%s, SHA1=%s', val_md5, val_sha1)
if not val_md5 or not val_sha1:
test.error('No MD5 or SHA1 value provided')
cmd_sha1 = params.get('cmd_sha1')
cmd_md5 = cmd_sha1 + ' MD5'
sha1 = vmcheck.session.cmd_output(
cmd_sha1, safe=True).strip().split('\n')[1].replace(' ', '')
md5 = vmcheck.session.cmd_output(
cmd_md5, safe=True).strip().split('\n')[1].replace(' ', '')
logging.info('Actual MD5=%s, SHA1=%s', md5, sha1)
if sha1 == val_sha1 and md5 == val_md5:
logging.info('MD5 and SHA1 are correct')
else:
log_fail('MD5 or SHA1 of rhsrvany.exe not correct')
def check_disk(vmcheck, count):
"""
Check if number of disks meets expectation
"""
logging.info('Expect number of disks: %d', count)
actual = vmcheck.session.cmd('lsblk |grep disk |wc -l').strip()
logging.info('Actual number of disks: %s', actual)
if int(actual) != count:
log_fail('Number of disks is wrong')
def check_result(result, status_error):
"""
Check virt-v2v command result
"""
libvirt.check_exit_status(result, status_error)
output = result.stdout + result.stderr
if not status_error and checkpoint != 'vdsm':
if output_mode == 'rhev':
if not utils_v2v.import_vm_to_ovirt(
params, address_cache, timeout=v2v_timeout):
test.fail('Import VM failed')
elif output_mode == 'libvirt':
try:
virsh.start(vm_name, debug=True, ignore_status=False)
except Exception as e:
test.fail('Start vm failed: %s', str(e))
# Check guest following the checkpoint document after convertion
logging.info('Checking common checkpoints for v2v')
vmchecker = VMChecker(test, params, env)
params['vmchecker'] = vmchecker
if params.get('skip_vm_check') != 'yes':
ret = vmchecker.run()
if len(ret) == 0:
logging.info("All common checkpoints passed")
else:
logging.info('Skip checking vm after conversion: %s' %
skip_reason)
# Check specific checkpoints
if checkpoint == 'console_xvc0':
check_grub_file(vmchecker.checker, 'console_xvc0')
if checkpoint in ('vnc_autoport', 'vnc_encrypt'):
vmchecker.check_graphics(params[checkpoint])
if checkpoint == 'sdl':
if output_mode == 'libvirt':
vmchecker.check_graphics({'type': 'vnc'})
elif output_mode == 'rhev':
vmchecker.check_graphics({'type': 'spice'})
if checkpoint == 'pv_with_regular_kernel':
check_kernel(vmchecker.checker)
if checkpoint in ['sound', 'pcspk']:
check_sound_card(vmchecker.checker, checkpoint)
if checkpoint == 'rhsrvany_md5':
check_rhsrvany_md5(vmchecker.checker)
if checkpoint == 'multidisk':
check_disk(vmchecker.checker, params['disk_count'])
log_check = utils_v2v.check_log(params, output)
if log_check:
log_fail(log_check)
# Merge 2 error lists
if params.get('vmchecker'):
error_list.extend(params['vmchecker'].errors)
# Virtio drivers will not be installed without virtio-win setup
if checkpoint == 'virtio_win_unset':
missing_list = params.get('missing').split(',')
expect_errors = ['Not find driver: ' + x for x in missing_list]
logging.debug('Expect errors: %s' % expect_errors)
logging.debug('Actual errors: %s' % error_list)
if set(error_list) == set(expect_errors):
error_list[:] = []
else:
logging.error('Virtio drivers not meet expectation')
if len(error_list):
test.fail('%d checkpoints failed: %s' %
(len(error_list), error_list))
try:
v2v_params = {
'hostname': xen_host,
'hypervisor': 'xen',
'main_vm': vm_name,
'v2v_opts': '-v -x',
'input_mode': 'libvirt',
'new_name': new_vm_name,
'password': xen_host_passwd,
'storage': params.get('output_storage', 'default'),
'network': params.get('network'),
'bridge': params.get('bridge'),
'target': params.get('target'),
'output_method': output_method,
'storage_name': storage_name,
'rhv_upload_opts': rhv_upload_opts
}
bk_xml = None
os.environ['LIBGUESTFS_BACKEND'] = 'direct'
# See man virt-v2v-input-xen(1)
process.run('update-crypto-policies --set LEGACY',
verbose=True,
ignore_status=True,
shell=True)
# Setup ssh-agent access to xen hypervisor
logging.info('set up ssh-agent access ')
xen_pubkey, xen_session = utils_v2v.v2v_setup_ssh_key(xen_host,
xen_host_user,
xen_host_passwd,
auto_close=False)
utils_misc.add_identities_into_ssh_agent()
if params.get('output_format'):
v2v_params.update({'output_format': params.get('output_format')})
# Build rhev related options
if output_mode == 'rhev':
# To RHV doesn't support 'qcow2' right now
v2v_params['output_format'] = 'raw'
# create different sasl_user name for different job
params.update({
'sasl_user':
params.get("sasl_user") + utils_misc.generate_random_string(3)
})
logging.info('sals user name is %s' % params.get("sasl_user"))
# Create SASL user on the ovirt host
user_pwd = "[['%s', '%s']]" % (params.get("sasl_user"),
params.get("sasl_pwd"))
v2v_sasl = utils_sasl.SASL(sasl_user_pwd=user_pwd)
v2v_sasl.server_ip = params.get("remote_ip")
v2v_sasl.server_user = params.get('remote_user')
v2v_sasl.server_pwd = params.get('remote_pwd')
v2v_sasl.setup(remote=True)
logging.debug('A SASL session %s was created', v2v_sasl)
if output_method == 'rhv_upload':
# Create password file for '-o rhv_upload' to connect to ovirt
with open(rhv_passwd_file, 'w') as f:
f.write(rhv_passwd)
# Copy ca file from ovirt to local
remote.scp_from_remote(ovirt_hostname, 22, 'root',
ovirt_engine_passwd, ovirt_ca_file_path,
local_ca_file_path)
# Create libvirt dir pool
if output_mode == 'libvirt':
pvt.pre_pool(pool_name, pool_type, pool_target, '')
uri = utils_v2v.Uri('xen').get_uri(xen_host)
virsh_dargs = {
'uri': uri,
'remote_ip': xen_host,
'remote_user': '******',
'remote_pwd': xen_host_passwd,
'auto_close': True,
'debug': True
}
virsh_instance = utils_v2v.wait_for(virsh.VirshPersistent,
**virsh_dargs)
logging.debug('A new virsh session %s was created', virsh_instance)
if not utils_v2v.wait_for(virsh_instance.domain_exists, name=vm_name):
test.error('VM %s not exists', vm_name)
if checkpoint in bk_list:
bk_xml = vm_xml.VMXML.new_from_inactive_dumpxml(
vm_name, virsh_instance=virsh_instance)
if checkpoint == 'guest_uuid':
uuid = virsh.domuuid(vm_name, uri=uri).stdout.strip()
v2v_params['main_vm'] = uuid
if checkpoint in ['format_convert', 'xvda_disk']:
# Get remote disk image path
blklist = virsh.domblklist(vm_name, uri=uri).stdout.split('\n')
logging.debug('domblklist %s:\n%s', vm_name, blklist)
for line in blklist:
if line.strip().startswith(('hda', 'vda', 'sda', 'xvda')):
params['remote_disk_image'] = line.split()[-1]
break
# Local path of disk image
params['img_path'] = data_dir.get_tmp_dir() + '/%s.img' % vm_name
if checkpoint == 'xvda_disk':
v2v_params['input_mode'] = 'disk'
v2v_params['hypervisor'] = 'kvm'
v2v_params.update({'input_file': params['img_path']})
# Copy remote image to local with scp
remote.scp_from_remote(xen_host, 22, xen_host_user,
xen_host_passwd,
params['remote_disk_image'],
params['img_path'])
if checkpoint == 'pool_uuid':
virsh.pool_start(pool_name)
pooluuid = virsh.pool_uuid(pool_name).stdout.strip()
v2v_params['storage'] = pooluuid
if checkpoint.startswith('vnc'):
vm_xml.VMXML.set_graphics_attr(vm_name, {'type': 'vnc'},
virsh_instance=virsh_instance)
if checkpoint == 'vnc_autoport':
params[checkpoint] = {'autoport': 'yes'}
vm_xml.VMXML.set_graphics_attr(vm_name,
params[checkpoint],
virsh_instance=virsh_instance)
elif checkpoint in ['vnc_encrypt', 'vnc_encrypt_warning']:
params[checkpoint] = {
'passwd': params.get('vnc_passwd', 'redhat')
}
vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(
vm_name, virsh_instance=virsh_instance)
vm_xml.VMXML.add_security_info(vmxml,
params[checkpoint]['passwd'],
virsh_instance=virsh_instance)
logging.debug(
virsh_instance.dumpxml(vm_name, extra='--security-info'))
if checkpoint.startswith('libguestfs_backend'):
value = checkpoint[19:]
if value == 'empty':
value = ''
logging.info('Set LIBGUESTFS_BACKEND to "%s"', value)
os.environ['LIBGUESTFS_BACKEND'] = value
if checkpoint == 'same_name':
logging.info('Convert guest and rename to %s', new_vm_name)
v2v_params.update({'new_name': new_vm_name})
if checkpoint == 'no_passwordless_SSH':
logging.info('Unset $SSH_AUTH_SOCK')
os.unsetenv('SSH_AUTH_SOCK')
if checkpoint in ['xml_without_image', 'format_convert']:
xml_file = os.path.join(data_dir.get_tmp_dir(), '%s.xml' % vm_name)
virsh.dumpxml(vm_name, to_file=xml_file, uri=uri)
v2v_params['hypervisor'] = 'kvm'
v2v_params['input_mode'] = 'libvirtxml'
v2v_params.update({'input_file': xml_file})
if params.get('img_path'):
cmd = "sed -i 's|%s|%s|' %s" % (params['remote_disk_image'],
params['img_path'], xml_file)
process.run(cmd)
logging.debug(process.run('cat %s' % xml_file).stdout_text)
if checkpoint == 'ssh_banner':
session = remote.remote_login("ssh", xen_host, "22", "root",
xen_host_passwd, "#")
ssh_banner_content = r'"# no default banner path\n' \
r'#Banner /path/banner file\n' \
r'Banner /etc/ssh/ssh_banner"'
logging.info('Create ssh_banner file')
session.cmd('echo -e %s > /etc/ssh/ssh_banner' %
ssh_banner_content)
logging.info('Content of ssh_banner file:')
logging.info(session.cmd_output('cat /etc/ssh/ssh_banner'))
logging.info('Restart sshd service on xen host')
session.cmd('service sshd restart')
if checkpoint.startswith('virtio_win'):
src_dir = params.get('virtio_win_dir')
dest_dir = os.path.join(data_dir.get_tmp_dir(), 'virtio-win')
iso_path = os.path.join(dest_dir, 'virtio-win.iso')
if not os.path.exists(dest_dir):
shutil.copytree(src_dir, dest_dir)
virtio_win_env = params.get('virtio_win_env', 'VIRTIO_WIN')
process.run('rpm -e virtio-win')
if process.run('rpm -q virtio-win',
ignore_status=True).exit_status == 0:
test.error('not removed')
if checkpoint.endswith('unset'):
logging.info('Unset env %s' % virtio_win_env)
os.unsetenv(virtio_win_env)
if checkpoint.endswith('custom'):
logging.info('Set env %s=%s' % (virtio_win_env, dest_dir))
os.environ[virtio_win_env] = dest_dir
if checkpoint.endswith('iso_mount'):
logging.info('Mount iso to /opt')
process.run('mount %s /opt' % iso_path)
os.environ[virtio_win_env] = '/opt'
if checkpoint.endswith('iso_file'):
logging.info('Set env %s=%s' % (virtio_win_env, iso_path))
os.environ[virtio_win_env] = iso_path
if checkpoint == 'cdrom':
xml = vm_xml.VMXML.new_from_inactive_dumpxml(
vm_name, virsh_instance=virsh_instance)
logging.debug(xml.xmltreefile)
disks = xml.get_disk_all()
logging.debug('Disks: %r', disks)
for disk in list(disks.values()):
# Check if vm has cdrom attached
if disk.get(
'device') == 'cdrom' and disk.find('source') is None:
test.error('No CDROM image attached')
if checkpoint == 'vdsm':
extra_pkg = params.get('extra_pkg')
logging.info('Install %s', extra_pkg)
utils_package.package_install(extra_pkg.split(','))
# Backup conf file for recovery
for conf in params['bk_conf'].strip().split(','):
logging.debug('Back up %s', conf)
shutil.copyfile(conf, conf + '.bk')
logging.info('Configure libvirt for vdsm')
process.run('vdsm-tool configure --force')
logging.info('Start vdsm service')
service_manager = service.Factory.create_generic_service()
service_manager.start('vdsmd')
# Setup user and password
user_pwd = "[['%s', '%s']]" % (params.get("sasl_user"),
params.get("sasl_pwd"))
v2v_sasl = utils_sasl.SASL(sasl_user_pwd=user_pwd)
v2v_sasl.server_ip = 'localhost'
v2v_sasl.server_user = params.get('sasl_server_user', 'root')
v2v_sasl.server_pwd = params.get('sasl_server_passwd')
v2v_sasl.setup()
logging.debug('A SASL session %s was created', v2v_sasl)
v2v_params['sasl_user'] = params.get("sasl_user")
v2v_params['sasl_pwd'] = params.get("sasl_pwd")
if checkpoint == 'multidisk':
params['disk_count'] = 0
blklist = virsh.domblklist(vm_name, uri=uri).stdout.split('\n')
logging.info(blklist)
for line in blklist:
if '/' in line:
params['disk_count'] += 1
logging.info('Total disks: %d', params['disk_count'])
# Execute virt-v2v
v2v_result = utils_v2v.v2v_cmd(v2v_params)
if new_vm_name:
vm_name = new_vm_name
params['main_vm'] = new_vm_name
check_result(v2v_result, status_error)
finally:
logging.info("Cleaning Environment")
# Cleanup constant files
utils_v2v.cleanup_constant_files(params)
# Restore crypto-policies to DEFAULT, the setting is impossible to be
# other values by default in testing envrionment.
process.run('update-crypto-policies --set DEFAULT',
verbose=True,
ignore_status=True,
shell=True)
if bk_xml:
bk_xml.sync(virsh_instance=virsh_instance)
if virsh_instance:
logging.debug('virsh session %s is closing', virsh_instance)
virsh_instance.close_session()
if params.get('vmchecker'):
params['vmchecker'].cleanup()
if output_mode == 'rhev' and v2v_sasl:
v2v_sasl.cleanup()
logging.debug('SASL session %s is closing', v2v_sasl)
v2v_sasl.close_session()
if checkpoint == 'vdsm':
logging.info('Stop vdsmd')
service_manager = service.Factory.create_generic_service()
service_manager.stop('vdsmd')
if params.get('extra_pkg'):
utils_package.package_remove(params['extra_pkg'].split(','))
for conf in params['bk_conf'].strip().split(','):
if os.path.exists(conf + '.bk'):
logging.debug('Recover %s', conf)
os.remove(conf)
shutil.move(conf + '.bk', conf)
logging.info('Restart libvirtd')
libvirtd = utils_libvirtd.Libvirtd()
libvirtd.restart()
logging.info('Start network "default"')
virsh.net_start('default')
virsh.undefine(vm_name)
if output_mode == 'libvirt':
pvt.cleanup_pool(pool_name, pool_type, pool_target, '')
if checkpoint == 'ssh_banner':
logging.info('Remove ssh_banner file')
session = remote.remote_login("ssh", xen_host, "22", "root",
xen_host_passwd, "#")
session.cmd('rm -f /etc/ssh/ssh_banner')
session.cmd('service sshd restart')
if checkpoint.startswith('virtio_win'):
utils_package.package_install(['virtio-win'])
utils_v2v.v2v_setup_ssh_key_cleanup(xen_session, xen_pubkey)
process.run('ssh-agent -k')
def remove_context_from_environment(plugin_context, prefix):
for key in plugin_context:
os.unsetenv(prefix + key)
def test_get_prod_label():
assert pelorus.get_prod_label() == pelorus.DEFAULT_PROD_LABEL
os.environ["PROD_LABEL"] = "changed"
assert pelorus.get_prod_label() == "changed"
os.unsetenv("PROD_LABEL")
def test_get_app_label():
assert pelorus.get_app_label() == pelorus.DEFAULT_APP_LABEL
os.environ["APP_LABEL"] = "changed"
assert pelorus.get_app_label() == "changed"
os.unsetenv("APP_LABEL")
def disableCache(self):
os.unsetenv('CACHEDIRECTORY')
os.environ['CACHEDIRECTORY'] = ""
def setUp(self):
self.setUpPyfakefs()
self.fs.add_real_file(
os.path.join(os.path.dirname(opencue.__file__), 'default.yaml'), read_only=True)
os.unsetenv('OPENCUE_CONFIG_FILE')
os.unsetenv('OPENCUE_CONF')
def read_env(var, default=None):
result = getenv(var, default)
log.debug("Cleaning env %s", var)
unsetenv(var)
return result
# Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
unicode_literals)
import os
import sys
# This __version__ assignment is parsed by setup.py; keep it in this form.
# Development versions end with ".dev" (suffix is added below).
__version__ = "0.3.dev"
__release__ = not __version__.endswith(".dev")
try:
from ._version import __githash__, __suffix__
except ImportError:
__githash__ = None
__suffix__ = "0"
if not __release__:
__version__ += __suffix__
del __suffix__
if sys.version_info >= (3, 3):
# OS X framework builds of Python 3.3 can not call other 3.3
# virtualenvs as a subprocess because `__PYENV_LAUNCHER__` is
# inherited.
if os.environ.get('__PYVENV_LAUNCHER__'):
os.unsetenv('__PYVENV_LAUNCHER__')
from . import plugin_manager
def _collectDebugInfo_environ(t):
dummyVars = ["WOOF_WOOFIE_%d" % x for x in xrange(4)]
global tag
tag = "XXX in os.environ"
try:
def f(name):
return "%s: %s" % (name, name in os.environ)
_infoX(f(x) for x in dummyVars)
except:
_info_exc()
tag = "os.environ[XXX]"
try:
def f(name):
try:
result = os.environ[name]
except:
result = _str_exc()
return "%s: %r" % (name, result)
_infoX(f(x) for x in dummyVars)
except:
_info_exc()
tag = "os.environ.get(XXX)"
try:
def f(name):
return "%s: %r" % (name, os.environ.get(name))
_infoX(f(x) for x in dummyVars)
except:
_info_exc()
tag = "os.getenv(XXX)"
try:
def f(name):
return "%s: %r" % (name, os.getenv(name))
_infoX(f(x) for x in dummyVars)
except:
_info_exc()
name = dummyVars[0]
value = "foo"
tag = "os.putenv(%s) to %r" % (name, value)
try:
results = []
_collect(results, "before", name, t)
os.putenv(name, value)
_collect(results, "after", name, t)
_infoX(results)
except:
_info_exc()
name = dummyVars[1]
value = "bar"
tag = "os.environ[%s] to %r" % (name, value)
try:
results = []
_collect(results, "before", name, t)
os.environ[name] = value
_collect(results, "after", name, t)
_infoX(results)
except:
_info_exc()
name = dummyVars[1]
value = "baz"
tag = "os.putenv(%s) to %r" % (name, value)
try:
results = []
_collect(results, "before", name, t)
os.putenv(name, value)
_collect(results, "after", name, t)
_infoX(results)
except:
_info_exc()
name = dummyVars[1]
value = ""
tag = "os.putenv(%s) to %r" % (name, value)
try:
results = []
_collect(results, "before", name, t)
os.putenv(name, value)
_collect(results, "after", name, t)
_infoX(results)
except:
_info_exc()
name = dummyVars[2]
value = "foo"
tag = "os.unsetenv(%s) from %r" % (name, value)
try:
results = []
os.environ[name] = value
_collect(results, "before", name, t)
os.unsetenv(name)
_collect(results, "after", name, t)
_infoX(results)
except:
_info_exc()
name = dummyVars[2]
value = "foo"
tag = "del os.environ[%s] from %r" % (name, value)
try:
results = []
os.environ[name] = value
_collect(results, "before", name, t)
del os.environ[name]
_collect(results, "after", name, t)
_infoX(results)
except:
_info_exc()
name = dummyVars[2]
value = "foo"
tag = "os.environ.pop(%s) from %r" % (name, value)
try:
results = []
os.environ[name] = value
_collect(results, "before", name, t)
os.environ.pop(name)
_collect(results, "after", name, t)
_infoX(results)
except:
_info_exc()
name = dummyVars[2]
value1 = "foo"
value2 = ""
tag = "os.environ[%s] to %r from %r" % (name, value2, value1)
try:
results = []
os.environ[name] = value1
_collect(results, "before", name, t)
os.environ[name] = value2
_collect(results, "after", name, t)
_infoX(results)
except:
_info_exc()
name = dummyVars[3]
value = '""'
tag = "os.environ[%s] to %r" % (name, value)
try:
results = []
_collect(results, "before", name, t)
os.environ[name] = value
_collect(results, "after", name, t)
_infoX(results)
except:
_info_exc()
def runSusTest(test, susdir, inputxml, compare_root, application, dbg_opt, max_parallelism, tests_to_do, tolerances, startFrom, varBucket, create_gs):
global startpath
global helperspath
global svn_revision
testname = getTestName(test)
np = float(getMPISize(test))
if (np > max_parallelism):
if np == 1.1:
print( "Skipping test %s because it requires mpi and max_parallism < 1.1" % testname )
return -1;
sus_options = varBucket[0]
do_plots = varBucket[1]
do_uda_comparison_test = tests_to_do[0]
do_memory_test = tests_to_do[1]
do_performance_test = tests_to_do[2]
compUda_RC = 0 # compare_uda return code
performance_RC = 0 # performance return code
memory_RC = 0 # memory return code
# turn off plotting option on restarts
if startFrom == "restart":
do_plots = 0
#__________________________________
# define resources limits
if environ['IGNORE_RLIMITS'] != "yes" :
#Kilo = 2**10
#Mega = 2**20
#Giga = 2**30
#resource.setrlimit(resource.RLIMIT_AS, (90*Mega,100*Mega) ) If we ever want to limit the memory
if dbg_opt == "dbg":
maxAllowRunTime = 1*60 # 30 minutes
else:
maxAllowRunTime = 1*60 # 15 minutes
resource.setrlimit(resource.RLIMIT_CPU, (maxAllowRunTime,maxAllowRunTime) )
#__________________________________
# turn on malloc_stats
if not do_memory_test :
unsetenv('MALLOC_STATS')
if getenv('MALLOC_STATS') == None:
MALLOCSTATS = ""
else:
MALLOCSTATS = "-x MALLOC_STATS"
#__________________________________
# Does mpirun command exist or has the environmental variable been set?
try :
MPIRUN = environ['MPIRUN'] # first try the environmental variable
except :
try:
MPIRUN = shutil.which("mpirun")
except:
print( "ERROR:runSusTests.py ")
print( " mpirun command was not found and the environmental variable MPIRUN was not set." )
print( " You must either add mpirun to your path, or set the 'MPIRUN' environment variable." )
exit (1)
MPIHEAD="%s -np" % MPIRUN #default
# pass in environmental variables to mpirun
if environ['OS'] == "Linux":
MPIHEAD="%s %s -np" % (MPIRUN, MALLOCSTATS)
# openmpi
rc = system("%s -x TERM echo 'hello' > /dev/null 2>&1" % MPIRUN)
if rc == 0:
MPIHEAD="%s %s -np" % (MPIRUN, MALLOCSTATS)
# mvapich
rc = system("%s -genvlist TERM echo 'hello' > /dev/null 2>&1" % MPIRUN)
if rc == 0:
MPIHEAD="%s -genvlist MALLOC_STATS -np" % MPIRUN
# if running performance tests, strip the output and checkpoints portions
# use the the input file local
if do_performance_test == 1:
localPath = "./"
inputxml = modUPS(localPath, inputxml,["<outputInterval>0</outputInterval>",
"<outputTimestepInterval>0</outputTimestepInterval>",
'<checkpoint cycle="0" interval="0"/>'])
# create a file in tmp/filename, copy it back
system("cp %s ." % inputxml)
inputxml = path.basename(inputxml)
SVN_OPTIONS = "-svnStat -svnDiff"
#SVN_OPTIONS = "" # When debugging, if you don't want to spend time waiting for SVN, uncomment this line.
command = "/usr/bin/time -p %s %s %s/sus %s %s " % (MPIHEAD, int(np), susdir, sus_options, SVN_OPTIONS)
mpimsg = " (mpi %s proc)" % (int(np))
time0 =time() #timer
#__________________________________
# setup input for sus
if startFrom == "restart":
print( "Running restart test ---%s--- %s at %s" % (testname, mpimsg, strftime( "%I:%M:%S")) )
susinput = "-restart ../*.uda.000 -t 0 -copy"
restart_text = " (restart)"
if startFrom == "inputFile":
print( "Running test ---%s--- %s at %s" % (testname, mpimsg, strftime( "%I:%M:%S")) )
susinput = "%s" % (inputxml)
restart_text = " "
if startFrom == "checkpoint":
print( "Running test from checkpoint ---%s--- %s at %s" % (testname, mpimsg, strftime( "%I:%M:%S")) )
susinput = "-restart %s/CheckPoints/%s/%s/*.uda.000" % (startpath,application,testname)
restart_text = " "
if startFrom == "postProcessUda":
print( "Running test from checkpoint ---%s--- %s at %s" % (testname, mpimsg, strftime( "%I:%M:%S")) )
susinput = "-postProcessUda %s/CheckPoints/%s/%s/*.uda.000" % (startpath,application,testname)
restart_text = " "
if do_memory_test == 1:
environ['MALLOC_STRICT'] = "set"
env = "%s,%s" % (environ['SCI_DEBUG'], "VarLabel:+") # append to the existing SCI_DEBUG
environ['SCI_DEBUG'] = env
if startFrom == "restart":
malloc_stats_file = "restart_malloc_stats"
else:
malloc_stats_file = "malloc_stats"
environ['MALLOC_STATS'] = malloc_stats_file
#__________________________________
# define failure messages
if getenv('OUTPUT_HTML') == "yes":
logpath = "%s/%s/%s-results/%s" % (environ['RT_URL'], dbg_opt, application, testname )
if startFrom == "restart":
logpath = logpath + "/restart"
sus_log_msg = '\t<A href=\"%s/sus.log.txt\">See sus.log</a> for details' % (logpath)
compare_msg = '\t<A href=\"%s/compare_sus_runs.log.txt\">See compare_sus_runs.log</A> for more comparison information.' % (logpath)
memory_msg = '\t<A href=\"%s/mem_leak_check.log.txt\">See mem_leak_check.log</a> for more comparison information.' % (logpath)
perf_msg = '\t<A href=\"%s/performance_check.log.txt\">See performance_check.log</a> for more comparison information.' % (logpath)
else:
logpath = "%s/%s-results/%s" % (startpath,application,testname)
sus_log_msg = '\tSee %s/sus.log.txt for details' % (logpath)
compare_msg = '\tSee %s/compare_sus_runs.log.txt for more comparison information.' % (logpath)
memory_msg = '\tSee %s/mem_leak_check.log.txt for more comparison information.' % (logpath)
perf_msg = '\tSee %s/performance_check.log.txt for more performance information.' % (logpath)
#__________________________________
# actually run the test!
short_cmd = command.replace(susdir+'/','')
print( "Command Line: %s %s" % (short_cmd, susinput) )
rc = system("env > sus.log.txt; %s %s >> sus.log.txt 2>&1" % (command, susinput))
# Check to see if an exception was thrown. (Use "grep -v 'cout'" to avoid false positive
# when source code line was that prints the exception is changed.)
# Did sus run to completion.
exception = system("grep -q 'Caught exception' sus.log.txt | grep -v cout");
try:
file = open('sus.log.txt', 'r')
lines = file.read()
susSuccess = re.findall("Sus: going down successfully", lines)
except Exception:
pass
if exception == 0:
print( "\t*** An exception was thrown ***" )
rc = -9
# determine path of replace_msg in 2 places to not have 2 different msgs.
replace_msg = "\tTo replace this test's goldStandards run:\n\t "
if startFrom == "restart":
chdir("..")
replace_msg = "%s%s/replace_gold_standard" % (replace_msg, getcwd())
chdir("restart")
else:
replace_msg = "%s%s/replace_gold_standard" % (replace_msg, getcwd())
replace_msg = "%s\n\t\t\tor\n\t %s/replace_all_GS\n" % (replace_msg,startpath)
#__________________________________
return_code = 0
if rc == 35072 or rc == 36608 :
print( "\t*** Test %s exceeded maximum allowable run time" % (testname) )
print( )
system("echo ' :%s: %s test exceeded maximum allowable run time' >> %s/%s-short.log" % (testname,restart_text,startpath,application))
return_code = 1
return return_code
elif rc != 0 or len(susSuccess) == 0 :
print( "\t*** Test %s failed to run to completion, (code %d)" % (testname, rc) )
if startFrom == "restart":
print( "\t\tMake sure the problem makes checkpoints before finishing" )
print( sus_log_msg )
print( "" )
system("echo ' :%s: %s test did not run to completion' >> %s/%s-short.log" % (testname,restart_text,startpath,application))
return_code = 1
return return_code
else:
# Sus completed successfully - now run memory, compare_uda and performance tests
# get the time from sus.log
# /usr/bin/time outputs 3 lines, the one called 'real' is what we want
# it is the third line from the bottom
# save this file independent of performance tests being done
print( "\tSuccessfully ran to completion" )
if startFrom == "restart":
ts_file = "restart_timestamp"
else:
ts_file = "timestamp"
system("tail -n3 sus.log.txt > %s" % ts_file)
#__________________________________
# performance test
if do_performance_test == 1:
print( "\tPerforming performance test on %s" % (date()) )
performance_RC = system("performance_check %s %s %s %s %s %s %s > performance_check.log.txt 2>&1" %
(testname, do_plots, ts_file, compare_root, helperspath, "sus.log.txt", svn_revision))
try:
short_message_file = open("performance_shortmessage.txt", 'r+', 500)
short_message = rstrip(short_message_file.readline(500))
except Exception:
short_message = ""
if performance_RC == 0:
print( "\tPerformance tests passed." )
if short_message != "":
print( "\t%s" % (short_message) )
elif performance_RC == 5 * 256:
print( "\t* Warning, no timestamp file created. No performance test performed." )
elif performance_RC == 2*256:
print( "\t*** Warning, test %s failed performance test." % (testname) )
if short_message != "":
print( "\t%s" % (short_message) )
print( perf_msg )
print( "%s" % replace_msg )
else:
print( "\tPerformance tests passed. (Note: no previous performace stats)." )
#__________________________________
# Memory leak test
if do_memory_test == 1:
memory_RC = system("mem_leak_check %s %d %s %s %s %s> mem_leak_check.log.txt 2>&1" %
(testname, do_plots, malloc_stats_file, compare_root, ".", helperspath))
try:
short_message_file = open("highwater_shortmessage.txt", 'r+', 500)
short_message = rstrip(short_message_file.readline(500))
except Exception:
short_message = ""
if memory_RC == 0:
print( "\tMemory leak tests passed." )
if short_message != "":
print( "\t%s" % (short_message) )
elif memory_RC == 5 * 256:
print( "\t*** ERROR, missing malloc_stats files. No memory tests performed." )
elif memory_RC == 256:
print( "\t*** ERROR, test %s failed memory leak test." % (testname) )
print( memory_msg )
# check that all VarLabels were deleted
rc = system("mem_leak_checkVarLabels sus.log.txt >> mem_leak_check.log.txt 2>&1")
elif memory_RC == 2*256:
print( "\t*** ERROR, test %s failed memory highwater test." % (testname) )
if short_message != "":
print( "\t%s" % (short_message) )
print( memory_msg )
print( "%s" % replace_msg )
else:
print( "\tMemory leak tests passed. (Note: no previous memory usage stats)." )
#__________________________________
# uda comparison
if do_uda_comparison_test == 1:
print( "\tComparing udas" )
if dbg_opt == "dbg":
environ['MALLOC_STATS'] = "compare_uda_malloc_stats"
abs_tol= tolerances[0]
rel_tol= tolerances[1]
compUda_RC = system("compare_sus_runs %s %s %s %s %s %s %s > compare_sus_runs.log.txt 2>&1" %
(testname, getcwd(), compare_root, susdir,abs_tol, rel_tol, create_gs))
if compUda_RC != 0:
if compUda_RC == 10 * 256:
print( "\t*** Input file(s) differs from the goldstandard" )
elif compUda_RC == 1 * 256:
print( "\t*** ERROR, test (%s) failed uda comparison, tolerances exceeded (%s)" % (testname, compUda_RC) )
print(compare_msg)
elif compUda_RC == 5*256:
print( "\t*** ERROR: test (%s) uda comparison aborted (%s)" % (testname, compUda_RC) )
print(compare_msg)
if startFrom != "restart":
(out,err,rc) = cmdline("tail -40 compare_sus_runs.log.txt | \
sed --silent /ERROR/,/ERROR/p | \
sed /'^$'/d | sed /'may not be compared'/,+1d") # clean out blank lines and cruft from the eror section
print( "\n\n%s\n" % out )
#print( "%s" % replace_msg )
elif compUda_RC == 65280: # (-1 return code)
print( "\tComparison tests passed. (Note: No dat files to compare.)" )
else:
print( "\tComparison tests passed. (Note: No previous gold standard.)" )
else:
print( "\tComparison tests passed." )
#__________________________________
# print error codes
# if comparison, memory, performance tests fail, return here, so mem_leak tests can run
if compUda_RC == 5*256 or compUda_RC == 1*256:
system("echo ' :%s: \t%s test failed comparison tests' >> %s/%s-short.log" % (testname,restart_text,startpath,application))
return_code = 2;
if performance_RC == 2*256:
system("echo ' :%s: \t%s test failed performance tests' >> %s/%s-short.log" % (testname,restart_text,startpath,application))
return_code = 2;
if memory_RC == 1*256 or memory_RC == 2*256 or memory_RC == 5*256:
system("echo ' :%s: \t%s test failed memory tests' >> %s/%s-short.log" % (testname,restart_text,startpath,application))
return_code = 2;
if return_code != 0:
# as the file is only created if a certain test fails, change the permissions here as we are certain the file exists
system("chmod gu+rw,a+r %s/%s-short.log > /dev/null 2>&1" % (startpath, application))
return return_code
parser.add_option('-b',
'--batch',
dest='batch',
action='store_true',
default=False,
help='Run in batch mode')
parser.add_option('-t',
'--tree',
dest='tree',
default='Beamspots',
help='Beamspot root-tree name')
(options, args) = parser.parse_args()
if options.batch:
os.unsetenv('DISPLAY')
from ROOT import TH1D, TCanvas
from ROOT import gSystem, gROOT, TFile, TH1D, TH2D, TCanvas, TTree, TChain
from ROOT import TGraphErrors
from ROOT import TMultiGraph
from ROOT import EColor
from ROOT import TLegend, TLine
from ROOT import TLatex, TGaxis
from ROOT import EColor
legends = [] # holder
pdglines = [] # holder
labels = []
ghists = [] # additional hists
failed = [] # list of failed plots
def unsetenv(key):
"""Compatibility wrapper for unsetting environment variables"""
os.environ.pop(key, None)
if hasattr(os, 'unsetenv'):
os.unsetenv(key)
def cleanup(self, tool_run_inf):
import os
os.chdir(self.__prev_dir)
os.unsetenv('COLINUX_PWD')
def _unset_envs(self):
if self._envs and self.args.runtime_backend != RuntimeBackendType.THREAD:
for k in self._envs.keys():
os.unsetenv(k)
ENV_KEY = "TEST_ENV_VAR" ENV_VALUE = "value" assert os.getenv(ENV_KEY) == None assert ENV_KEY not in os.environ assert os.getenv(ENV_KEY, 5) == 5 os.environ[ENV_KEY] = ENV_VALUE assert ENV_KEY in os.environ assert os.getenv(ENV_KEY) == ENV_VALUE del os.environ[ENV_KEY] assert ENV_KEY not in os.environ assert os.getenv(ENV_KEY) == None if os.name == "posix": os.putenv(ENV_KEY, ENV_VALUE) os.unsetenv(ENV_KEY) assert os.getenv(ENV_KEY) == None assert os.curdir == "." assert os.pardir == ".." assert os.extsep == "." if os.name == "nt": assert os.sep == "\\" assert os.linesep == "\r\n" assert os.altsep == "/" assert os.pathsep == ";" else: assert os.sep == "/" assert os.linesep == "\n" assert os.altsep == None
def os_func():
'''
操作系统模块
该模块下的方法,对各个版本的兼容不明确,须谨慎使用.
测试版本: Python:3.6.1 Windows:Windows10,64bit
'''
# === 系统 ===
strs = os.name # 当前系统: Linux'posix' / Windows'nt' / 'ce' / 'java'
strs = os.sep # 分隔符 \\ (windows:\\ linux:/)
strs = os.pathsep # path变量分隔符 ; (windows:; linux::)
strs = os.linesep # 换行分隔符 \r\n (windows:/r/n linux:\n)
dics = os.environ # 查看系统环境变量(字典)
strs = os.getenv("Path", default=-1) # 读取环境变量, 没有返回None
os.putenv("Path", "C:\\python") # 添加环境变量 (windows无效)
os.unsetenv("Path") # 删除环境变量 (windows不可用)
strs = os.getlogin() # 当前登录的用户名
num = os.getpid() # 当前进程PID
num = os.system("cmd") # 执行操作系统命令, 返回0/1(0执行正确;1执行错误)
strs = os.popen("dir").read() # 执行系统命令,并去读结果
tups = os.times() # 当前进程时间(user用户 / system系统 / children_user子用户(windews=0) / children_system子系统(windews=0) / elapsed过去时间)
bytes = os.urandom(10) # n字节用以加密的随机字符
num = os.cpu_count() # CUP数量
# === 进程 ===
os.abort() # 结束进程
# execl(file, *args) / execle / execlp / execlpe / execv / execve / execvp / execvpe // 运行新程序替换当前进程
os.execl(r"C:\python", 'python', 'hello.py', 'i') # (windows执行失败)
os._exit(0) # 退出
os.kill(8480, signal.SIGTERM) # (系统) 终止进程(需要导入:signal) SIGINT (终止进程) / SIGTERM (终止进程) / SIGKILL (终止进程) / SIGALRM (闹钟信号)
# === 文件 / 文件夹 ===
strs = os.getcwd() # 当前路径
bytes = os.getcwdb() # 当前路径
os.chdir(r"C:\Windows") # 切换路径
strs = os.curdir # 当前目录 .
strs = os.pardir # 上级目录 ..
strs = os.sep # 路径分隔符 ('/' or '\\')
bytes = os.fsencode(r"C:\c.obj") # (编码) 文件路径字符串转为bytes类型 => b'C:\\c.obj'
strs = os.fsdecode(b"C:\c.obj") # (解码) 文件路径转为strs类型 => 'C:\\c.obj'
# chmod(path, mode, *, dir_fd=None, follow_symlinks=True)
os.chmod(r"C:\python\hello.py", 777) # 修改模式
os.link("file.txt", "file.txt.link") # 创建硬链接
os.symlink("file.txt", "file.txt.link") # 创建软链接 (windows执行失败)
lists = os.listdir() # 所有文件和文件夹(列表) ".""..""D:"
# lstat(path, *, dir_fd=None)
tups = os.lstat(r"C:\c.obj") # 状态信息列表
boolean = os.access(r"C:\c.obj", os.F_OK) # (文件/文件夹) 权限测试 (mode: F_OK存在? / R_OK可读? / W_OK可写? / X_OK可执行?)
# scandir(path='.') // DirEntry迭代器, 文件目录
lists = os.scandir()
# st_atime (访问时间) / st_ctime (修改时间) / st_mtime (模式修改时间) / st_size (大小(字节bytes))
# st_uid (用户ID) / st_gid (用户组ID)
# st_ino (inode) / st_mode (模式) / st_dev (设备) / st_nlink (硬链接)
# count = cpu_count() # (系统) CPU线程数(非核数)
tups = os.stat(".") # 获取状态信息, 返回stat_result对象
# utime(path, times=None, *, ns=None, dir_fd=None, follow_symlinks=True) // 更新文件的访问和修改时间
num = os.utime(r"C:\c.obj")
# walk(top, topdown=True, onerror=None, followlinks=False) // 根目录(top)遍历目录树,返回迭代器 (dirpath, dirnames[], filenames[]).
root, dirnames, filenames = os.walk(r"C:\python")
os.removedirs(r"c:\python") # 删除多个文件夹 (windews删除多个文件夹失败,单个成功)
# mkdir(path, mode=0o777, *, dir_fd=None) // 创建单个目录, 文件夹已存在,抛 FileExistsError 异常
os.mkdir("test")
# makedirs(name, mode=0o777, exist_ok=False) // 创建(多个)文件夹
os.makedirs(r"./t1/t2/t3")
os.rmdir("test") # 删除单个目录
# mknod(path, mode=0o600, device=0, *, dir_fd=None) // 创建空文件 (windows不可用)
os.mknod("test.txt")
# remove(path, *, dir_fd=None)
os.remove("test.txt") # 删除文件
# rename(src, dst, *, src_dir_fd=None, dst_dir_fd=None)
os.rename("text.txt", "file.txt") # 重命名
os.renames("text.txt", "file.txt")
# replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None) // 重命名, 存在则替换
os.replace("text.txt", "file.txt")
tups = os.stat(r"text.txt") # 文件属性
# === 文件读写 ===
# open(path, flags, mode=0o777, *, dir_fd=None) // 打开文件 fd:文件描述符
fd = os.open(r"C:\c.obj", os.O_RDWR | os.O_CREAT)
readfd, writefd = os.pipe() # 打开管道,返回读取,写入 (windows失败)
# fdopen(fd, *args, **kwargs) // 打开 (windews写入失败,读取""字符)
f = os.fdopen(readfd)
os.read(fd, 150) # 读取
os.write(fd, "String".encode("utf-8")) # 写入
os.fsync(fd) # 强行写入
os.ftruncate(fd, 100) # 裁剪文件
bytes = os.lseek(fd, 10, os.SEEK_SET) # 设置指针 SEEK_SET(0开始) SEEK_CUR(1当前位置) SEEK_END(2末尾)
fd_temp = os.dup(fd) # 副本
boolean = os.isatty(fd) # 是否是tty设备
stat = os.fstat(fd) # 状态信息
strs = os.device_encoding(fd) # 返回终端字符编码,非终端None
os.close(fd) # 关闭
os.closerange(fd, fd) # 关闭并忽略错误
# === DirEntry ===
for dir in os.scandir():
strs = dir.name # 文件名
strs = dir.path # 完整路径名
num = dir.inode() # inode编号
boolean = dir.is_dir() # 是否是文件夹
boolean = dir.is_file() # 是否是文件
boolean = dir.is_symlink() # 是否是链接
tups = dir.stat() # 状态信息的stat_result对象
vm_xml.VMXML.add_security_info(
vmxml, params[checkpoint]['passwd'],
virsh_instance=virsh_instance)
logging.debug(virsh_instance.dumpxml(vm_name, extra='--security-info'))
elif checkpoint.startswith('libguestfs_backend'):
value = checkpoint[19:]
if value == 'empty':
value = ''
logging.info('Set LIBGUESTFS_BACKEND to "%s"', value)
os.environ['LIBGUESTFS_BACKEND'] = value
elif checkpoint == 'same_name':
logging.info('Convert guest and rename to %s', new_vm_name)
v2v_params.update({'new_name': new_vm_name})
elif checkpoint == 'no_passwordless_SSH':
logging.info('Unset $SSH_AUTH_SOCK')
os.unsetenv('SSH_AUTH_SOCK')
elif checkpoint == 'xml_without_image':
xml_file = os.path.join(data_dir.get_tmp_dir(), '%s.xml' % vm_name)
virsh.dumpxml(vm_name, to_file=xml_file, uri=uri)
v2v_params['hypervisor'] = 'kvm'
v2v_params['input_mode'] = 'libvirtxml'
v2v_params.update({'input_file': xml_file})
elif checkpoint == 'ssh_banner':
session = remote.remote_login("ssh", xen_host, "22", "root",
xen_host_passwd, "#")
ssh_banner_content = r'"# no default banner path\n' \
r'#Banner /path/banner file\n' \
r'Banner /etc/ssh/ssh_banner"'
logging.info('Create ssh_banner file')
session.cmd('echo -e %s > /etc/ssh/ssh_banner' % ssh_banner_content)
logging.info('Content of ssh_banner file:')
def __init__(self, *args, **kwargs):
hasattr(sys, "frozen") and os.putenv("_MEIPASS2", sys._MEIPASS + os.sep)
try: super(Popen, self).__init__(*args, **kwargs)
finally: hasattr(sys, "frozen") and os.unsetenv("_MEIPASS2")
def tearDownClass(cls):
elasticsearch_delete_index('*' + Config.test_index_suffix.value)
Config.test_index_suffix.restore()
cls.server.shutdown()
os.unsetenv('DSS_ES_PORT')
if not os.getcwd() in sys.path:
sys.path = [os.getcwd()] + sys.path
if not '' in sys.path:
sys.path = [''] + sys.path
sys.ps1 = 'pyroot> '
fhistory = os.path.expanduser('~/.pyroot.history')
## interface setup as appropriate
if runBatch:
# in batch there is no need for stdin
if os.isatty(sys.stdin.fileno()):
os.close(sys.stdin.fileno())
if 'PYTHONINSPECT' in os.environ:
os.unsetenv('PYTHONINSPECT')
else:
os.environ['PYTHONINSPECT'] = '1'
# readline support
import rlcompleter, readline
readline.parse_and_bind('tab: complete')
readline.parse_and_bind('set show-all-if-ambiguous On')
# history support
if os.path.exists(fhistory):
readline.read_history_file(fhistory)
readline.set_history_length(1024)
del readline, rlcompleter
def application(self, env, start_response):
# find map name
map_name = env['PATH_INFO'].split('/')[-1]
# text debug
if self.debug >= 1:
self._logging(1, "-" * 30)
for key in self.MAPSERV_ENV:
if key in env:
os.environ[key] = env[key]
self._logging(1, "{0}='{1}'".format(key, env[key]))
else:
os.unsetenv(key)
if self.debug >= 2:
self._logging(2, "QUERY_STRING=(")
for q_str in env['QUERY_STRING'].split('&'):
self._logging(2, " {},".format(q_str))
self._logging(2, ")")
self._logging(1, "-" * 30)
# serialization & response
while True:
if not self.maps.has_key(map_name):
# serialization
map_ = self.serializer(map_name)
if map_ and map_name not in self.invariable_name:
self.maps[map_name] = map_
else:
self._logging(
0,
"ERROR: Map:'{}' is not serialized".format(map_name))
resp_status = '404 Not Found'
resp_type = [('Content-type', 'text/plain')]
start_response(resp_status, resp_type)
return [b'MAP:\'{}\' not found'.format(map_name)]
else:
# response (mono or multi)
if self.multi and self.maps[map_name]['multi']:
que = Queue()
proc = Process(target=self.maps[map_name]['request'],
name='response',
args=(env, self.maps[map_name]['content'],
que))
proc.start()
response = que.get()
proc.join()
else:
response = self.maps[map_name]['request'](
env, self.maps[map_name]['content'])
# fin response
if response:
if self.maps[map_name]['timestamp'] != 0:
self.maps[map_name]['timestamp'] = int(time.time())
resp_status = '200 OK'
resp_type = [('Content-type', response[0])]
start_response(resp_status, resp_type)
return [response[1]]
else:
self._logging(0,
"ERROR: Resource:{} error".format(map_name))
resp_status = '500 Server Error'
resp_type = [('Content-type', 'text/plain')]
start_response(resp_status, resp_type)
return [b'Resource:{} error'.format(map_name)]
def unset_environment_vars(self):
if self._envs and self.args.runtime != 'thread':
for k in self._envs.keys():
os.unsetenv(k)
def test_default_cache(self):
os.unsetenv('PYSERINI_CACHE')
SimpleSearcher.from_prebuilt_index('cacm')
self.assertTrue(
os.path.exists(os.path.expanduser('~/.cache/pyserini/indexes')))
def test_revision_lang_de(self):
os.putenv('LANG', 'de_DE.UTF-8')
self._revision()
os.unsetenv('LANG')
def setUp(self):
# Test without access to a Streams install
self._si = os.environ.get('STREAMS_INSTALL')
if self._si is not None:
os.unsetenv('STREAMS_INSTALL')
