def set_version():
p_out = output_subprocess_Popen([nvcc_path, "--version"])
ver_line = decode(p_out[0]).strip().split("\n")[-1]
build, version = ver_line.split(",")[1].strip().split()
assert build == "release"
global nvcc_version
nvcc_version = version
def set_version():
p_out = output_subprocess_Popen([nvcc_path, '--version'])
ver_line = decode(p_out[0]).strip().split('\n')[-1]
build, version = ver_line.split(',')[1].strip().split()
assert build == 'release'
global nvcc_version
nvcc_version = version
def run_command(command, **params):
# Define a primitive function that executes a given command in
# a controlled environment and collects all output.
"""A simple primitive for executing a given command and
grabing its output.
:param command: a list of string representing the command
to be executed;
:param **params: optional named parametrs passed to
output_subprocess_Popen()."""
stdout, stderr, exit_code = '', '', -1
try:
stdout, stderr, exit_code = output_subprocess_Popen(command, **params)
except Exception as e:
stderr = str(e)
return exit_code, decode(stdout), decode(stderr)
def run(stdout, stderr, argv, theano_nose, batch_size, time_profile,
display_batch_output):
# Setting aside current working directory for later saving
sav_dir = os.getcwd()
# The first argument is the called script.
argv = argv[1:]
# It seems safer to fully regenerate the list of tests on each call.
if os.path.isfile('.noseids'):
os.remove('.noseids')
# Collect test IDs.
print("""\
####################
# COLLECTING TESTS #
####################""")
stdout.flush()
stderr.flush()
dummy_in = open(os.devnull)
# We need to call 'python' on Windows, because theano-nose is not a
# native Windows app; and it does not hurt to call it on Unix.
# Using sys.executable, so that the same Python version is used.
python = sys.executable
rval = subprocess.call(
([python, theano_nose, '--collect-only', '--with-id'] + argv),
stdin=dummy_in.fileno(),
stdout=stdout.fileno(),
stderr=stderr.fileno())
stdout.flush()
stderr.flush()
assert rval == 0
noseids_file = '.noseids'
with open(noseids_file, 'rb') as f:
data = pickle.load(f)
ids = data['ids']
n_tests = len(ids)
if n_tests == 0:
raise Exception("0 test selected")
assert n_tests == max(ids)
# Standard batch testing is called for
if not time_profile:
failed = set()
print("""\
###################################
# RUNNING TESTS IN BATCHES OF %s #
###################################""" % batch_size)
# When `display_batch_output` is False, we suppress all output because
# we want the user to focus only on the failed tests, which are re-run
# (with output) below.
dummy_out = open(os.devnull, 'w')
for test_id in xrange(1, n_tests + 1, batch_size):
stdout.flush()
stderr.flush()
test_range = list(
range(test_id, min(test_id + batch_size, n_tests + 1)))
cmd = ([python, theano_nose, '--with-id'] +
list(map(str, test_range)) + argv)
subprocess_extra_args = dict(stdin=dummy_in.fileno())
if not display_batch_output:
# Use quiet mode in nosetests.
cmd.append('-q')
# Suppress all output.
subprocess_extra_args.update(
dict(stdout=dummy_out.fileno(), stderr=dummy_out.fileno()))
t0 = time.time()
subprocess.call(cmd, **subprocess_extra_args)
t1 = time.time()
# Recover failed test indices from the 'failed' field of the
# '.noseids' file. We need to do it after each batch because
# otherwise this field may get erased. We use a set because it
# seems like it is not systematically erased though, and we want
# to avoid duplicates.
with open(noseids_file, 'rb') as f:
failed = failed.union(pickle.load(f)['failed'])
print('%s%% done in %.3fs (failed: %s)' %
((test_range[-1] * 100) // n_tests, t1 - t0, len(failed)))
# Sort for cosmetic purpose only.
failed = sorted(failed)
if failed:
# Re-run only failed tests
print("""\
################################
# RE-RUNNING FAILED TESTS ONLY #
################################""")
stdout.flush()
stderr.flush()
subprocess.call(
([python, theano_nose, '-v', '--with-id'] + failed + argv),
stdin=dummy_in.fileno(),
stdout=stdout.fileno(),
stderr=stderr.fileno())
stdout.flush()
stderr.flush()
return 0
else:
print("""\
####################
# ALL TESTS PASSED #
####################""")
# Time-profiling is called for
else:
print("""\
########################################
# RUNNING TESTS IN TIME-PROFILING MODE #
########################################""")
# finds first word of list l containing string s
def getIndexOfFirst(l, s):
for pos, word in enumerate(l):
if s in word:
return pos
# finds last word of list l containing string s
def getIndexOfLast(l, s):
for pos, word in enumerate(reversed(l)):
if s in word:
return (len(l) - pos - 1)
# iterating through tests
# initializing master profiling list and raw log
prof_master_nosort = []
dummy_out = open(os.devnull, 'w')
path_rawlog = os.path.join(sav_dir, 'timeprof_rawlog')
stamp = str(datetime.datetime.now()) + '\n\n'
f_rawlog = open(path_rawlog, 'w')
f_rawlog.write('TIME-PROFILING OF THEANO\'S NOSETESTS'
' (raw log)\n\n' + stamp)
f_rawlog.flush()
stamp = str(datetime.datetime.now()) + '\n\n'
fields = ('Fields: computation time; nosetests sequential id;'
' test name; parent class (if any); outcome\n\n')
path_nosort = os.path.join(sav_dir, 'timeprof_nosort')
# probably this part can be extracted for function with many args
with open(path_nosort, 'w') as f_nosort:
# begin of saving nosort
f_nosort.write('TIME-PROFILING OF THEANO\'S NOSETESTS'
' (by sequential id)\n\n' + stamp + fields)
f_nosort.flush()
for test_floor in xrange(1, n_tests + 1, batch_size):
for test_id in xrange(
test_floor, min(test_floor + batch_size, n_tests + 1)):
# Print the test we will start in the raw log to help
# debug tests that are too long.
f_rawlog.write(
"\n%s Will run test #%d %s\n" %
(time.ctime(), test_id, data["ids"][test_id]))
f_rawlog.flush()
p_out = output_subprocess_Popen(
([python, theano_nose, '-v', '--with-id'] +
[str(test_id)] + argv + ['--disabdocstring']))
# the previous option calls a custom Nosetests plugin
# precluding automatic sustitution of doc. string for
# test name in display
# (see class 'DisabDocString' in file theano-nose)
# recovering and processing data from pipe
err = p_out[1]
# print the raw log
f_rawlog.write(err)
f_rawlog.flush()
# parsing the output
l_err = err.split()
try:
pos_id = getIndexOfFirst(l_err, '#')
prof_id = l_err[pos_id]
pos_dot = getIndexOfFirst(l_err, '...')
prof_test = ''
for s in l_err[pos_id + 1:pos_dot]:
prof_test += s + ' '
if 'OK' in err:
pos_ok = getIndexOfLast(l_err, 'OK')
if len(l_err) == pos_ok + 1:
prof_time = float(l_err[pos_ok - 1][0:-1])
prof_pass = '******'
elif 'SKIP' in l_err[pos_ok + 1]:
prof_time = 0.
prof_pass = '******'
elif 'KNOWNFAIL' in l_err[pos_ok + 1]:
prof_time = float(l_err[pos_ok - 1][0:-1])
prof_pass = '******'
else:
prof_time = 0.
prof_pass = '******'
else:
prof_time = 0.
prof_pass = '******'
except Exception:
prof_time = 0
prof_id = '#' + str(test_id)
prof_test = ('FAILED PARSING, see raw log for details'
' on test')
prof_pass = ''
prof_tuple = (prof_time, prof_id, prof_test, prof_pass)
# appending tuple to master list
prof_master_nosort.append(prof_tuple)
# write the no sort file
s_nosort = ((str(prof_tuple[0]) + 's').ljust(10) + " " +
prof_tuple[1].ljust(7) + " " + prof_tuple[2] +
prof_tuple[3] + "\n")
f_nosort.write(s_nosort)
f_nosort.flush()
print('%s%% time-profiled' % ((test_id * 100) // n_tests))
f_rawlog.close()
# sorting tests according to running-time
prof_master_sort = sorted(prof_master_nosort,
key=lambda test: test[0],
reverse=True)
# saving results to readable files
path_sort = os.path.join(sav_dir, 'timeprof_sort')
with open(path_sort, 'w') as f_sort:
f_sort.write('TIME-PROFILING OF THEANO\'S NOSETESTS'
' (sorted by computation time)\n\n' + stamp +
fields)
for i in xrange(len(prof_master_nosort)):
s_sort = ((str(prof_master_sort[i][0]) + 's').ljust(10) +
" " + prof_master_sort[i][1].ljust(7) + " " +
prof_master_sort[i][2] + prof_master_sort[i][3] +
"\n")
f_sort.write(s_sort)
import socket
import subprocess
import sys
import textwrap
import numpy
import theano
from theano.configparser import config, AddConfigVar, ConfigParam, StrParam
from theano.gof.utils import flatten
from theano.misc.windows import output_subprocess_Popen
_logger = logging.getLogger("theano.gof.compiledir")
try:
p_out = output_subprocess_Popen([theano.config.cxx, '-dumpversion'])
gcc_version_str = p_out[0].strip().decode()
except OSError:
# Typically means gcc cannot be found.
gcc_version_str = 'GCC_NOT_FOUND'
def local_bitwidth():
"""
Return 32 for 32bit arch, 64 for 64bit arch
By "architecture", we mean the size of memory pointers (size_t in C),
*not* the size of long int, as it can be different.
"""
# Note that according to Python documentation, `platform.architecture()` is
# not reliable on OS X with universal binaries.
def run(stdout, stderr, argv, theano_nose, batch_size, time_profile, display_batch_output):
# Setting aside current working directory for later saving
sav_dir = os.getcwd()
# The first argument is the called script.
argv = argv[1:]
# It seems safer to fully regenerate the list of tests on each call.
if os.path.isfile(".noseids"):
os.remove(".noseids")
# Collect test IDs.
print(
"""\
####################
# COLLECTING TESTS #
####################"""
)
stdout.flush()
stderr.flush()
dummy_in = open(os.devnull)
# We need to call 'python' on Windows, because theano-nose is not a
# native Windows app; and it does not hurt to call it on Unix.
# Using sys.executable, so that the same Python version is used.
python = sys.executable
rval = subprocess.call(
([python, theano_nose, "--collect-only", "--with-id"] + argv),
stdin=dummy_in.fileno(),
stdout=stdout.fileno(),
stderr=stderr.fileno(),
)
stdout.flush()
stderr.flush()
assert rval == 0
noseids_file = ".noseids"
with open(noseids_file, "rb") as f:
data = cPickle.load(f)
ids = data["ids"]
n_tests = len(ids)
if n_tests == 0:
raise Exception("0 test selected")
assert n_tests == max(ids)
# Standard batch testing is called for
if not time_profile:
failed = set()
print(
"""\
###################################
# RUNNING TESTS IN BATCHES OF %s #
###################################"""
% batch_size
)
# When `display_batch_output` is False, we suppress all output because
# we want the user to focus only on the failed tests, which are re-run
# (with output) below.
dummy_out = open(os.devnull, "w")
for test_id in xrange(1, n_tests + 1, batch_size):
stdout.flush()
stderr.flush()
test_range = range(test_id, min(test_id + batch_size, n_tests + 1))
cmd = [python, theano_nose, "--with-id"] + map(str, test_range) + argv
subprocess_extra_args = dict(stdin=dummy_in.fileno())
if not display_batch_output:
# Use quiet mode in nosetests.
cmd.append("-q")
# Suppress all output.
subprocess_extra_args.update(dict(stdout=dummy_out.fileno(), stderr=dummy_out.fileno()))
t0 = time.time()
subprocess.call(cmd, **subprocess_extra_args)
t1 = time.time()
# Recover failed test indices from the 'failed' field of the
# '.noseids' file. We need to do it after each batch because
# otherwise this field may get erased. We use a set because it
# seems like it is not systematically erased though, and we want
# to avoid duplicates.
with open(noseids_file, "rb") as f:
failed = failed.union(cPickle.load(f)["failed"])
print("%s%% done in %.3fs (failed: %s)" % ((test_range[-1] * 100) // n_tests, t1 - t0, len(failed)))
# Sort for cosmetic purpose only.
failed = sorted(failed)
if failed:
# Re-run only failed tests
print(
"""\
################################
# RE-RUNNING FAILED TESTS ONLY #
################################"""
)
stdout.flush()
stderr.flush()
subprocess.call(
([python, theano_nose, "-v", "--with-id"] + failed + argv),
stdin=dummy_in.fileno(),
stdout=stdout.fileno(),
stderr=stderr.fileno(),
)
stdout.flush()
stderr.flush()
return 0
else:
print(
"""\
####################
# ALL TESTS PASSED #
####################"""
)
# Time-profiling is called for
else:
print(
"""\
########################################
# RUNNING TESTS IN TIME-PROFILING MODE #
########################################"""
)
# finds first word of list l containing string s
def getIndexOfFirst(l, s):
for pos, word in enumerate(l):
if s in word:
return pos
# finds last word of list l containing string s
def getIndexOfLast(l, s):
for pos, word in enumerate(reversed(l)):
if s in word:
return len(l) - pos - 1
# iterating through tests
# initializing master profiling list and raw log
prof_master_nosort = []
prof_rawlog = []
dummy_out = open(os.devnull, "w")
path_rawlog = os.path.join(sav_dir, "timeprof_rawlog")
stamp = str(datetime.datetime.now()) + "\n\n"
f_rawlog = open(path_rawlog, "w")
f_rawlog.write("TIME-PROFILING OF THEANO'S NOSETESTS" " (raw log)\n\n" + stamp)
f_rawlog.flush()
stamp = str(datetime.datetime.now()) + "\n\n"
fields = "Fields: computation time; nosetests sequential id;" " test name; parent class (if any); outcome\n\n"
path_nosort = os.path.join(sav_dir, "timeprof_nosort")
f_nosort = open(path_nosort, "w")
f_nosort.write("TIME-PROFILING OF THEANO'S NOSETESTS" " (by sequential id)\n\n" + stamp + fields)
f_nosort.flush()
for test_floor in xrange(1, n_tests + 1, batch_size):
for test_id in xrange(test_floor, min(test_floor + batch_size, n_tests + 1)):
# Print the test we will start in the raw log to help
# debug tests that are too long.
f_rawlog.write("\n%s Will run test #%d %s\n" % (time.ctime(), test_id, data["ids"][test_id]))
f_rawlog.flush()
p_out = output_subprocess_Popen(
([python, theano_nose, "-v", "--with-id"] + [str(test_id)] + argv + ["--disabdocstring"])
)
# the previous option calls a custom Nosetests plugin
# precluding automatic sustitution of doc. string for
# test name in display
# (see class 'DisabDocString' in file theano-nose)
# recovering and processing data from pipe
err = p_out[1]
# print the raw log
f_rawlog.write(err)
f_rawlog.flush()
# parsing the output
l_err = err.split()
try:
pos_id = getIndexOfFirst(l_err, "#")
prof_id = l_err[pos_id]
pos_dot = getIndexOfFirst(l_err, "...")
prof_test = ""
for s in l_err[pos_id + 1 : pos_dot]:
prof_test += s + " "
if "OK" in err:
pos_ok = getIndexOfLast(l_err, "OK")
if len(l_err) == pos_ok + 1:
prof_time = float(l_err[pos_ok - 1][0:-1])
prof_pass = "******"
elif "SKIP" in l_err[pos_ok + 1]:
prof_time = 0.0
prof_pass = "******"
elif "KNOWNFAIL" in l_err[pos_ok + 1]:
prof_time = float(l_err[pos_ok - 1][0:-1])
prof_pass = "******"
else:
prof_time = 0.0
prof_pass = "******"
else:
prof_time = 0.0
prof_pass = "******"
except Exception:
prof_time = 0
prof_id = "#" + str(test_id)
prof_test = "FAILED PARSING, see raw log for details" " on test"
prof_pass = ""
prof_tuple = (prof_time, prof_id, prof_test, prof_pass)
# appending tuple to master list
prof_master_nosort.append(prof_tuple)
# write the no sort file
s_nosort = (
(str(prof_tuple[0]) + "s").ljust(10)
+ " "
+ prof_tuple[1].ljust(7)
+ " "
+ prof_tuple[2]
+ prof_tuple[3]
+ "\n"
)
f_nosort.write(s_nosort)
f_nosort.flush()
print("%s%% time-profiled" % ((test_id * 100) // n_tests))
f_rawlog.close()
# sorting tests according to running-time
prof_master_sort = sorted(prof_master_nosort, key=lambda test: test[0], reverse=True)
# saving results to readable files
path_sort = os.path.join(sav_dir, "timeprof_sort")
f_sort = open(path_sort, "w")
f_sort.write("TIME-PROFILING OF THEANO'S NOSETESTS" " (sorted by computation time)\n\n" + stamp + fields)
for i in xrange(len(prof_master_nosort)):
s_sort = (
(str(prof_master_sort[i][0]) + "s").ljust(10)
+ " "
+ prof_master_sort[i][1].ljust(7)
+ " "
+ prof_master_sort[i][2]
+ prof_master_sort[i][3]
+ "\n"
)
f_sort.write(s_sort)
f_nosort.close()
f_sort.close()
import subprocess
import sys
import textwrap
import numpy
import theano
from theano.configparser import config, AddConfigVar, ConfigParam, StrParam
from theano.gof.utils import flatten
from theano.misc.windows import output_subprocess_Popen
_logger = logging.getLogger("theano.gof.compiledir")
try:
p_out = output_subprocess_Popen([theano.config.cxx, '-dumpversion'])
gcc_version_str = p_out[0].strip().decode()
except OSError:
# Typically means gcc cannot be found.
gcc_version_str = 'GCC_NOT_FOUND'
def local_bitwidth():
"""
Return 32 for 32bit arch, 64 for 64bit arch
By "architecture", we mean the size of memory pointers (size_t in C),
*not* the size of long int, as it can be different.
"""
# Note that according to Python documentation, `platform.architecture()` is
# not reliable on OS X with universal binaries.
import subprocess
import sys
import textwrap
import numpy
import theano
from theano.configparser import config, AddConfigVar, ConfigParam, StrParam
from theano.gof.utils import flatten
from theano.misc.windows import output_subprocess_Popen
_logger = logging.getLogger("theano.gof.compiledir")
try:
p_out = output_subprocess_Popen(['g++', '-dumpversion'])
gcc_version_str = p_out[0].strip().decode()
except OSError:
# Typically means gcc cannot be found.
gcc_version_str = 'GCC_NOT_FOUND'
def local_bitwidth():
"""
Return 32 for 32bit arch, 64 for 64bit arch
By "architecture", we mean the size of memory pointers (size_t in C),
*not* the size of long int, as it can be different.
"""
# Note that according to Python documentation, `platform.architecture()` is
# not reliable on OS X with universal binaries.
import socket
import subprocess
import sys
import textwrap
import numpy
import theano
from theano.configparser import config, AddConfigVar, ConfigParam, StrParam
from theano.gof.utils import flatten
from theano.misc.windows import output_subprocess_Popen
_logger = logging.getLogger("theano.gof.compiledir")
try:
p_out = output_subprocess_Popen(['g++', '-dumpversion'])
gcc_version_str = p_out[0].strip().decode()
except OSError:
# Typically means gcc cannot be found.
gcc_version_str = 'GCC_NOT_FOUND'
def local_bitwidth():
"""
Return 32 for 32bit arch, 64 for 64bit arch
By "architecture", we mean the size of memory pointers (size_t in C),
*not* the size of long int, as it can be different.
"""
# Note that according to Python documentation, `platform.architecture()` is
# not reliable on OS X with universal binaries.
