def set_version(): p = call_subprocess_Popen([nvcc_path, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.wait() s = p.stdout.readlines()[-1].split(',')[1].strip().split() assert s[0] == 'release' global nvcc_version nvcc_version = s[1]
def set_version(): p = call_subprocess_Popen([nvcc_path, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.wait() ver_line = decode(p.stdout.readlines()[-1]) build, version = ver_line.split(',')[1].strip().split() assert build == 'release' global nvcc_version nvcc_version = version
def test_gxx_support(): default_openmp = True try: code = """ #include <omp.h> int main( int argc, const char* argv[] ) { int res[10]; for(int i=0; i < 10; i++){ res[i] = i; } } """ fd, path = tempfile.mkstemp(suffix='.c', prefix='test_omp_') dummy_stdin = open(os.devnull) try: os.write(fd, code) os.close(fd) fd = None proc = call_subprocess_Popen(['g++', '-fopenmp', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=dummy_stdin.fileno()) proc.wait() if proc.returncode != 0: default_openmp = False finally: del dummy_stdin # Ensure `fd` is closed before we remove the temporary file. try: if fd is not None: os.close(fd) finally: os.remove(path) except OSError, e: return False
# The way to get FAST_RUN_NOGC is with the flag 'linker=c|py_nogc'. # The old all capital letter way of working is deprecated as it is not # scalable. # Also, please be careful not to modify the first item in the enum when adding # new modes, since it is the default mode. AddConfigVar('mode', "Default compilation mode", EnumStr('Mode', 'ProfileMode', 'DebugMode', 'FAST_RUN', 'FAST_COMPILE', 'PROFILE_MODE', 'DEBUG_MODE'), in_c_key=False) enum = EnumStr("g++", "") # Test whether or not g++ is present: disable C code if it is not. try: rc = call_subprocess_Popen(['g++', '-v']) except OSError: enum = EnumStr("") rc = 1 AddConfigVar('cxx', "The C++ compiler to use. Currently only g++ is" " supported, but supporting additional compilers should not be " "too difficult. " "If it is empty, no C++ code is compiled.", enum, in_c_key=False) del enum if rc == 0 and config.cxx != "": # Keep the default linker the same as the one for the mode FAST_RUN AddConfigVar('linker',
# The old all capital letter way of working is deprecated as it is not # scalable. # Also, please be careful not to modify the first item in the enum when adding # new modes, since it is the default mode. AddConfigVar( 'mode', "Default compilation mode", EnumStr('Mode', 'ProfileMode', 'DebugMode', 'FAST_RUN', 'FAST_COMPILE', 'PROFILE_MODE', 'DEBUG_MODE'), in_c_key=False) param = "g++" # Test whether or not g++ is present: disable C code if it is not. try: rc = call_subprocess_Popen(['g++', '-v']) except OSError: rc = 1 if rc != 0: param = "" # On Mac we test for 'clang++' and use it by default if sys.platform == 'darwin': try: rc = call_subprocess_Popen(['clang++', '-v']) if rc == 0: param = "clang++" except OSError: pass
AddConfigVar('mode', "Default compilation mode", EnumStr('Mode', 'ProfileMode', 'DebugMode', 'FAST_RUN', 'FAST_COMPILE', 'PROFILE_MODE', 'DEBUG_MODE'), in_c_key=False) enum = EnumStr("g++", "") # Test whether or not g++ is present: disable C code if it is not. # Using the dummy file descriptor below is a workaround for a crash experienced # in an unusual Python 2.4.4 Windows environment with the default stdin=None. dummy_stdin = open(os.devnull) try: try: rc = call_subprocess_Popen(['g++', '-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=dummy_stdin).wait() except OSError: rc = 1 finally: dummy_stdin.close() del dummy_stdin if rc == 0: # Keep the default linker the same as the one for the mode FAST_RUN AddConfigVar('linker', ("Default linker used if the theano flags mode is Mode " "or ProfileMode"), EnumStr('cvm', 'c|py', 'py', 'c', 'c|py_nogc', 'c&py', 'vm', 'vm_nogc', 'cvm_nogc'), in_c_key=False) else:
import numpy import theano from theano.configparser import config, AddConfigVar, ConfigParam, StrParam from theano.gof.utils import flatten from theano.misc.windows import call_subprocess_Popen # Using the dummy file descriptors below is a workaround for a crash # experienced in an unusual Python 2.4.4 Windows environment with the default # None values. dummy_in = open(os.devnull) dummy_err = open(os.devnull, 'w') p = None try: p = call_subprocess_Popen(['g++', '-dumpversion'], stdout=subprocess.PIPE, stdin=dummy_in.fileno(), stderr=dummy_err.fileno()) p.wait() gcc_version_str = p.stdout.readline().strip() except OSError: # Typically means gcc cannot be found. gcc_version_str = 'GCC_NOT_FOUND' del p del dummy_in del dummy_err compiledir_format_dict = {"platform": platform.platform(), "processor": platform.processor(), "python_version": platform.python_version(), "theano_version": theano.__version__, "numpy_version": numpy.__version__,
"mode", "Default compilation mode", EnumStr("Mode", "ProfileMode", "DebugMode", "FAST_RUN", "FAST_COMPILE", "PROFILE_MODE", "DEBUG_MODE"), in_c_key=False, ) enum = EnumStr("g++", "") # Test whether or not g++ is present: disable C code if it is not. # Using the dummy file descriptor below is a workaround for a crash experienced # in an unusual Python 2.4.4 Windows environment with the default stdin=None. dummy_stdin = open(os.devnull) try: try: rc = call_subprocess_Popen( ["g++", "-v"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=dummy_stdin ).wait() except OSError: rc = 1 finally: dummy_stdin.close() del dummy_stdin if rc == 0: # Keep the default linker the same as the one for the mode FAST_RUN AddConfigVar( "linker", ("Default linker used if the theano flags mode is Mode " "or ProfileMode"), EnumStr("cvm", "c|py", "py", "c", "c|py_nogc", "c&py", "vm", "vm_nogc", "cvm_nogc"), in_c_key=False, ) else:
def run(stdout, stderr, argv, theano_nose, batch_size, time_profile, display_batch_output): # Setting aside current working directory for later saving sav_dir = os.getcwd() # The first argument is the called script. argv = argv[1:] # It seems safer to fully regenerate the list of tests on each call. if os.path.isfile('.noseids'): os.remove('.noseids') # Collect test IDs. print """\ #################### # COLLECTING TESTS # ####################""" stdout.flush() stderr.flush() dummy_in = open(os.devnull) # We need to call 'python' on Windows, because theano-nose is not a # native Windows app; and it does not hurt to call it on Unix. # Using sys.executable, so that the same Python version is used. python = sys.executable rval = subprocess.call( ([python, theano_nose, '--collect-only', '--with-id'] + argv), stdin=dummy_in.fileno(), stdout=stdout.fileno(), stderr=stderr.fileno()) stdout.flush() stderr.flush() assert rval == 0 noseids_file = '.noseids' data = cPickle.load(open(noseids_file, 'rb')) ids = data['ids'] n_tests = len(ids) assert n_tests == max(ids) # Standard batch testing is called for if not time_profile: failed = set() print """\ ################################### # RUNNING TESTS IN BATCHES OF %s # ###################################""" % batch_size # When `display_batch_output` is False, we suppress all output because # we want the user to focus only on the failed tests, which are re-run # (with output) below. dummy_out = open(os.devnull, 'w') for test_id in xrange(1, n_tests + 1, batch_size): stdout.flush() stderr.flush() test_range = range(test_id, min(test_id + batch_size, n_tests + 1)) cmd = ([python, theano_nose, '--with-id'] + map(str, test_range) + argv) subprocess_extra_args = dict(stdin=dummy_in.fileno()) if not display_batch_output: # Use quiet mode in nosetests. cmd.append('-q') # Suppress all output. subprocess_extra_args.update(dict( stdout=dummy_out.fileno(), stderr=dummy_out.fileno())) subprocess.call(cmd, **subprocess_extra_args) # Recover failed test indices from the 'failed' field of the # '.noseids' file. We need to do it after each batch because # otherwise this field may get erased. We use a set because it # seems like it is not systematically erased though, and we want # to avoid duplicates. failed = failed.union(cPickle.load(open(noseids_file, 'rb')) ['failed']) print '%s%% done (failed: %s)' % ((test_range[-1] * 100) // n_tests, len(failed)) # Sort for cosmetic purpose only. failed = sorted(failed) if failed: # Re-run only failed tests print """\ ################################ # RE-RUNNING FAILED TESTS ONLY # ################################""" stdout.flush() stderr.flush() subprocess.call( ([python, theano_nose, '-v', '--with-id'] + failed + argv), stdin=dummy_in.fileno(), stdout=stdout.fileno(), stderr=stderr.fileno()) stdout.flush() stderr.flush() return 0 else: print """\ #################### # ALL TESTS PASSED # ####################""" # Time-profiling is called for else: print """\ ######################################## # RUNNING TESTS IN TIME-PROFILING MODE # ########################################""" # finds first word of list l containing string s def getIndexOfFirst(l, s): for pos, word in enumerate(l): if s in word: return pos # finds last word of list l containing string s def getIndexOfLast(l, s): for pos, word in enumerate(reversed(l)): if s in word: return (len(l) - pos - 1) # iterating through tests # initializing master profiling list and raw log prof_master_nosort = [] prof_rawlog = [] dummy_out = open(os.devnull, 'w') path_rawlog = os.path.join(sav_dir, 'timeprof_rawlog') stamp = str(datetime.datetime.now()) + '\n\n' f_rawlog = open(path_rawlog, 'w') f_rawlog.write('TIME-PROFILING OF THEANO\'S NOSETESTS' ' (raw log)\n\n' + stamp) f_rawlog.flush() stamp = str(datetime.datetime.now()) + '\n\n' fields = ('Fields: computation time; nosetests sequential id;' ' test name; parent class (if any); outcome\n\n') path_nosort = os.path.join(sav_dir, 'timeprof_nosort') f_nosort = open(path_nosort, 'w') f_nosort.write('TIME-PROFILING OF THEANO\'S NOSETESTS' ' (by sequential id)\n\n' + stamp + fields) f_nosort.flush() for test_floor in xrange(1, n_tests + 1, batch_size): for test_id in xrange(test_floor, min(test_floor + batch_size, n_tests + 1)): # Print the test we will start in the raw log to help # debug tests that are too long. f_rawlog.write("\n%s Will run test #%d %s\n" % ( time.ctime(), test_id, data["ids"][test_id])) f_rawlog.flush() proc = call_subprocess_Popen( ([python, theano_nose, '-v', '--with-id'] + [str(test_id)] + argv + ['--disabdocstring']), # the previous option calls a custom Nosetests plugin # precluding automatic sustitution of doc. string for # test name in display # (see class 'DisabDocString' in file theano-nose) stderr=subprocess.PIPE, stdout=dummy_out.fileno()) # recovering and processing data from pipe err = proc.stderr.read() # print the raw log f_rawlog.write(err) f_rawlog.flush() # parsing the output l_err = err.split() try: pos_id = getIndexOfFirst(l_err, '#') prof_id = l_err[pos_id] pos_dot = getIndexOfFirst(l_err, '...') prof_test = '' for s in l_err[pos_id + 1: pos_dot]: prof_test += s + ' ' if 'OK' in err: pos_ok = getIndexOfLast(l_err, 'OK') if len(l_err) == pos_ok + 1: prof_time = float(l_err[pos_ok - 1][0:-1]) prof_pass = '******' elif 'SKIP' in l_err[pos_ok + 1]: prof_time = 0. prof_pass = '******' elif 'KNOWNFAIL' in l_err[pos_ok + 1]: prof_time = float(l_err[pos_ok - 1][0:-1]) prof_pass = '******' else: prof_time = 0. prof_pass = '******' else: prof_time = 0. prof_pass = '******' except Exception: prof_time = 0 prof_id = '#' + str(test_id) prof_test = ('FAILED PARSING, see raw log for details' ' on test') prof_pass = '' prof_tuple = (prof_time, prof_id, prof_test, prof_pass) # appending tuple to master list prof_master_nosort.append(prof_tuple) # write the no sort file s_nosort = ((str(prof_tuple[0]) + 's').ljust(10) + " " + prof_tuple[1].ljust(7) + " " + prof_tuple[2] + prof_tuple[3] + "\n") f_nosort.write(s_nosort) f_nosort.flush() print '%s%% time-profiled' % ((test_id * 100) // n_tests) f_rawlog.close() # sorting tests according to running-time prof_master_sort = sorted(prof_master_nosort, key=lambda test: test[0], reverse=True) # saving results to readable files path_sort = os.path.join(sav_dir, 'timeprof_sort') f_sort = open(path_sort, 'w') f_sort.write('TIME-PROFILING OF THEANO\'S NOSETESTS' ' (sorted by computation time)\n\n' + stamp + fields) for i in xrange(len(prof_master_nosort)): s_sort = ((str(prof_master_sort[i][0]) + 's').ljust(10) + " " + prof_master_sort[i][1].ljust(7) + " " + prof_master_sort[i][2] + prof_master_sort[i][3] + "\n") f_sort.write(s_sort) f_nosort.close() f_sort.close()
import numpy import theano from theano.configparser import config, AddConfigVar, ConfigParam, StrParam from theano.gof.utils import flatten from theano.misc.windows import call_subprocess_Popen # Using the dummy file descriptors below is a workaround for a crash # experienced in an unusual Python 2.4.4 Windows environment with the default # None values. dummy_err = open(os.devnull, 'w') p = None try: p = call_subprocess_Popen(['g++', '-dumpversion'], stdout=subprocess.PIPE, stderr=dummy_err.fileno()) p.wait() gcc_version_str = p.stdout.readline().strip().decode() except OSError: # Typically means gcc cannot be found. gcc_version_str = 'GCC_NOT_FOUND' del p del dummy_err def local_bitwidth(): """ Return 32 for 32bit arch, 64 for 64bit arch By "architecture", we mean the size of memory pointers (size_t in C),
def run(stdout, stderr, argv, theano_nose, batch_size, time_profile, display_batch_output): # Setting aside current working directory for later saving sav_dir = os.getcwd() # The first argument is the called script. argv = argv[1:] # It seems safer to fully regenerate the list of tests on each call. if os.path.isfile(".noseids"): os.remove(".noseids") # Collect test IDs. print """\ #################### # COLLECTING TESTS # ####################""" stdout.flush() stderr.flush() dummy_in = open(os.devnull) # We need to call 'python' on Windows, because theano-nose is not a # native Windows app; and it does not hurt to call it on Unix. # Using sys.executable, so that the same Python version is used. python = sys.executable rval = subprocess.call( ([python, theano_nose, "--collect-only", "--with-id"] + argv), stdin=dummy_in.fileno(), stdout=stdout.fileno(), stderr=stderr.fileno(), ) stdout.flush() stderr.flush() assert rval == 0 noseids_file = ".noseids" data = cPickle.load(open(noseids_file, "rb")) ids = data["ids"] n_tests = len(ids) assert n_tests == max(ids) # Standard batch testing is called for if not time_profile: failed = set() print """\ ################################### # RUNNING TESTS IN BATCHES OF %s # ###################################""" % batch_size # When `display_batch_output` is False, we suppress all output because # we want the user to focus only on the failed tests, which are re-run # (with output) below. dummy_out = open(os.devnull, "w") for test_id in xrange(1, n_tests + 1, batch_size): stdout.flush() stderr.flush() test_range = range(test_id, min(test_id + batch_size, n_tests + 1)) cmd = [python, theano_nose, "--with-id"] + map(str, test_range) + argv subprocess_extra_args = dict(stdin=dummy_in.fileno()) if not display_batch_output: # Use quiet mode in nosetests. cmd.append("-q") # Suppress all output. subprocess_extra_args.update(dict(stdout=dummy_out.fileno(), stderr=dummy_out.fileno())) subprocess.call(cmd, **subprocess_extra_args) # Recover failed test indices from the 'failed' field of the # '.noseids' file. We need to do it after each batch because # otherwise this field may get erased. We use a set because it # seems like it is not systematically erased though, and we want # to avoid duplicates. failed = failed.union(cPickle.load(open(noseids_file, "rb"))["failed"]) print "%s%% done (failed: %s)" % ((test_range[-1] * 100) // n_tests, len(failed)) # Sort for cosmetic purpose only. failed = sorted(failed) if failed: # Re-run only failed tests print """\ ################################ # RE-RUNNING FAILED TESTS ONLY # ################################""" stdout.flush() stderr.flush() subprocess.call( ([python, theano_nose, "-v", "--with-id"] + failed + argv), stdin=dummy_in.fileno(), stdout=stdout.fileno(), stderr=stderr.fileno(), ) stdout.flush() stderr.flush() return 0 else: print """\ #################### # ALL TESTS PASSED # ####################""" # Time-profiling is called for else: print """\ ######################################## # RUNNING TESTS IN TIME-PROFILING MODE # ########################################""" # finds first word of list l containing string s def getIndexOfFirst(l, s): for pos, word in enumerate(l): if s in word: return pos # finds last word of list l containing string s def getIndexOfLast(l, s): for pos, word in enumerate(reversed(l)): if s in word: return len(l) - pos - 1 # iterating through tests # initializing master profiling list and raw log prof_master_nosort = [] prof_rawlog = [] dummy_out = open(os.devnull, "w") path_rawlog = os.path.join(sav_dir, "timeprof_rawlog") stamp = str(datetime.datetime.now()) + "\n\n" f_rawlog = open(path_rawlog, "w") f_rawlog.write("TIME-PROFILING OF THEANO'S NOSETESTS" " (raw log)\n\n" + stamp) f_rawlog.flush() stamp = str(datetime.datetime.now()) + "\n\n" fields = "Fields: computation time; nosetests sequential id;" " test name; parent class (if any); outcome\n\n" path_nosort = os.path.join(sav_dir, "timeprof_nosort") f_nosort = open(path_nosort, "w") f_nosort.write("TIME-PROFILING OF THEANO'S NOSETESTS" " (by sequential id)\n\n" + stamp + fields) f_nosort.flush() for test_floor in xrange(1, n_tests + 1, batch_size): for test_id in xrange(test_floor, min(test_floor + batch_size, n_tests + 1)): # Print the test we will start in the raw log to help # debug tests that are too long. f_rawlog.write("\n%s Will run test #%d %s\n" % (time.ctime(), test_id, data["ids"][test_id])) f_rawlog.flush() proc = call_subprocess_Popen( ([python, theano_nose, "-v", "--with-id"] + [str(test_id)] + argv + ["--disabdocstring"]), # the previous option calls a custom Nosetests plugin # precluding automatic sustitution of doc. string for # test name in display # (see class 'DisabDocString' in file theano-nose) stderr=subprocess.PIPE, stdout=dummy_out.fileno(), ) # recovering and processing data from pipe err = proc.stderr.read() # print the raw log f_rawlog.write(err) f_rawlog.flush() # parsing the output l_err = err.split() try: pos_id = getIndexOfFirst(l_err, "#") prof_id = l_err[pos_id] pos_dot = getIndexOfFirst(l_err, "...") prof_test = "" for s in l_err[pos_id + 1 : pos_dot]: prof_test += s + " " if "OK" in err: pos_ok = getIndexOfLast(l_err, "OK") if len(l_err) == pos_ok + 1: prof_time = float(l_err[pos_ok - 1][0:-1]) prof_pass = "******" elif "SKIP" in l_err[pos_ok + 1]: prof_time = 0.0 prof_pass = "******" elif "KNOWNFAIL" in l_err[pos_ok + 1]: prof_time = float(l_err[pos_ok - 1][0:-1]) prof_pass = "******" else: prof_time = 0.0 prof_pass = "******" else: prof_time = 0.0 prof_pass = "******" except Exception: prof_time = 0 prof_id = "#" + str(test_id) prof_test = "FAILED PARSING, see raw log for details" " on test" prof_pass = "" prof_tuple = (prof_time, prof_id, prof_test, prof_pass) # appending tuple to master list prof_master_nosort.append(prof_tuple) # write the no sort file s_nosort = ( (str(prof_tuple[0]) + "s").ljust(10) + " " + prof_tuple[1].ljust(7) + " " + prof_tuple[2] + prof_tuple[3] + "\n" ) f_nosort.write(s_nosort) f_nosort.flush() print "%s%% time-profiled" % ((test_id * 100) // n_tests) f_rawlog.close() # sorting tests according to running-time prof_master_sort = sorted(prof_master_nosort, key=lambda test: test[0], reverse=True) # saving results to readable files path_sort = os.path.join(sav_dir, "timeprof_sort") f_sort = open(path_sort, "w") f_sort.write("TIME-PROFILING OF THEANO'S NOSETESTS" " (sorted by computation time)\n\n" + stamp + fields) for i in xrange(len(prof_master_nosort)): s_sort = ( (str(prof_master_sort[i][0]) + "s").ljust(10) + " " + prof_master_sort[i][1].ljust(7) + " " + prof_master_sort[i][2] + prof_master_sort[i][3] + "\n" ) f_sort.write(s_sort) f_nosort.close() f_sort.close()
# new modes, since it is the default mode. AddConfigVar( "mode", "Default compilation mode", EnumStr("Mode", "ProfileMode", "DebugMode", "FAST_RUN", "FAST_COMPILE", "PROFILE_MODE", "DEBUG_MODE"), in_c_key=False, ) enum = EnumStr("g++", "") # Test whether or not g++ is present: disable C code if it is not. # Using the dummy file descriptor below is a workaround for a crash experienced # in an unusual Python 2.4.4 Windows environment with the default stdin=None. dummy_stdin = open(os.devnull) try: call_subprocess_Popen("g++", stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=dummy_stdin.fileno()) # Keep the default linker the same as the one for the mode FAST_RUN AddConfigVar( "linker", ("Default linker used if the theano flags mode is Mode " "or ProfileMode"), EnumStr("cvm", "c|py", "py", "c", "c|py_nogc", "c&py", "vm", "vm_nogc", "cvm_nogc"), in_c_key=False, ) except OSError: # g++ is not present, linker should default to python only AddConfigVar( "linker", ("Default linker used if the theano flags mode is Mode " "or ProfileMode"), EnumStr("py", "vm", "vm_nogc"), in_c_key=False, )