def set_version(): p_out = output_subprocess_Popen([nvcc_path, "--version"]) ver_line = decode(p_out[0]).strip().split("\n")[-1] build, version = ver_line.split(",")[1].strip().split() assert build == "release" global nvcc_version nvcc_version = version
def set_version(): p_out = output_subprocess_Popen([nvcc_path, '--version']) ver_line = decode(p_out[0]).strip().split('\n')[-1] build, version = ver_line.split(',')[1].strip().split() assert build == 'release' global nvcc_version nvcc_version = version
def run_command(command, **params): # Define a primitive function that executes a given command in # a controlled environment and collects all output. """A simple primitive for executing a given command and grabing its output. :param command: a list of string representing the command to be executed; :param **params: optional named parametrs passed to output_subprocess_Popen().""" stdout, stderr, exit_code = '', '', -1 try: stdout, stderr, exit_code = output_subprocess_Popen(command, **params) except Exception as e: stderr = str(e) return exit_code, decode(stdout), decode(stderr)
def run(stdout, stderr, argv, theano_nose, batch_size, time_profile, display_batch_output): # Setting aside current working directory for later saving sav_dir = os.getcwd() # The first argument is the called script. argv = argv[1:] # It seems safer to fully regenerate the list of tests on each call. if os.path.isfile('.noseids'): os.remove('.noseids') # Collect test IDs. print("""\ #################### # COLLECTING TESTS # ####################""") stdout.flush() stderr.flush() dummy_in = open(os.devnull) # We need to call 'python' on Windows, because theano-nose is not a # native Windows app; and it does not hurt to call it on Unix. # Using sys.executable, so that the same Python version is used. python = sys.executable rval = subprocess.call( ([python, theano_nose, '--collect-only', '--with-id'] + argv), stdin=dummy_in.fileno(), stdout=stdout.fileno(), stderr=stderr.fileno()) stdout.flush() stderr.flush() assert rval == 0 noseids_file = '.noseids' with open(noseids_file, 'rb') as f: data = pickle.load(f) ids = data['ids'] n_tests = len(ids) if n_tests == 0: raise Exception("0 test selected") assert n_tests == max(ids) # Standard batch testing is called for if not time_profile: failed = set() print("""\ ################################### # RUNNING TESTS IN BATCHES OF %s # ###################################""" % batch_size) # When `display_batch_output` is False, we suppress all output because # we want the user to focus only on the failed tests, which are re-run # (with output) below. dummy_out = open(os.devnull, 'w') for test_id in xrange(1, n_tests + 1, batch_size): stdout.flush() stderr.flush() test_range = list( range(test_id, min(test_id + batch_size, n_tests + 1))) cmd = ([python, theano_nose, '--with-id'] + list(map(str, test_range)) + argv) subprocess_extra_args = dict(stdin=dummy_in.fileno()) if not display_batch_output: # Use quiet mode in nosetests. cmd.append('-q') # Suppress all output. subprocess_extra_args.update( dict(stdout=dummy_out.fileno(), stderr=dummy_out.fileno())) t0 = time.time() subprocess.call(cmd, **subprocess_extra_args) t1 = time.time() # Recover failed test indices from the 'failed' field of the # '.noseids' file. We need to do it after each batch because # otherwise this field may get erased. We use a set because it # seems like it is not systematically erased though, and we want # to avoid duplicates. with open(noseids_file, 'rb') as f: failed = failed.union(pickle.load(f)['failed']) print('%s%% done in %.3fs (failed: %s)' % ((test_range[-1] * 100) // n_tests, t1 - t0, len(failed))) # Sort for cosmetic purpose only. failed = sorted(failed) if failed: # Re-run only failed tests print("""\ ################################ # RE-RUNNING FAILED TESTS ONLY # ################################""") stdout.flush() stderr.flush() subprocess.call( ([python, theano_nose, '-v', '--with-id'] + failed + argv), stdin=dummy_in.fileno(), stdout=stdout.fileno(), stderr=stderr.fileno()) stdout.flush() stderr.flush() return 0 else: print("""\ #################### # ALL TESTS PASSED # ####################""") # Time-profiling is called for else: print("""\ ######################################## # RUNNING TESTS IN TIME-PROFILING MODE # ########################################""") # finds first word of list l containing string s def getIndexOfFirst(l, s): for pos, word in enumerate(l): if s in word: return pos # finds last word of list l containing string s def getIndexOfLast(l, s): for pos, word in enumerate(reversed(l)): if s in word: return (len(l) - pos - 1) # iterating through tests # initializing master profiling list and raw log prof_master_nosort = [] dummy_out = open(os.devnull, 'w') path_rawlog = os.path.join(sav_dir, 'timeprof_rawlog') stamp = str(datetime.datetime.now()) + '\n\n' f_rawlog = open(path_rawlog, 'w') f_rawlog.write('TIME-PROFILING OF THEANO\'S NOSETESTS' ' (raw log)\n\n' + stamp) f_rawlog.flush() stamp = str(datetime.datetime.now()) + '\n\n' fields = ('Fields: computation time; nosetests sequential id;' ' test name; parent class (if any); outcome\n\n') path_nosort = os.path.join(sav_dir, 'timeprof_nosort') # probably this part can be extracted for function with many args with open(path_nosort, 'w') as f_nosort: # begin of saving nosort f_nosort.write('TIME-PROFILING OF THEANO\'S NOSETESTS' ' (by sequential id)\n\n' + stamp + fields) f_nosort.flush() for test_floor in xrange(1, n_tests + 1, batch_size): for test_id in xrange( test_floor, min(test_floor + batch_size, n_tests + 1)): # Print the test we will start in the raw log to help # debug tests that are too long. f_rawlog.write( "\n%s Will run test #%d %s\n" % (time.ctime(), test_id, data["ids"][test_id])) f_rawlog.flush() p_out = output_subprocess_Popen( ([python, theano_nose, '-v', '--with-id'] + [str(test_id)] + argv + ['--disabdocstring'])) # the previous option calls a custom Nosetests plugin # precluding automatic sustitution of doc. string for # test name in display # (see class 'DisabDocString' in file theano-nose) # recovering and processing data from pipe err = p_out[1] # print the raw log f_rawlog.write(err) f_rawlog.flush() # parsing the output l_err = err.split() try: pos_id = getIndexOfFirst(l_err, '#') prof_id = l_err[pos_id] pos_dot = getIndexOfFirst(l_err, '...') prof_test = '' for s in l_err[pos_id + 1:pos_dot]: prof_test += s + ' ' if 'OK' in err: pos_ok = getIndexOfLast(l_err, 'OK') if len(l_err) == pos_ok + 1: prof_time = float(l_err[pos_ok - 1][0:-1]) prof_pass = '******' elif 'SKIP' in l_err[pos_ok + 1]: prof_time = 0. prof_pass = '******' elif 'KNOWNFAIL' in l_err[pos_ok + 1]: prof_time = float(l_err[pos_ok - 1][0:-1]) prof_pass = '******' else: prof_time = 0. prof_pass = '******' else: prof_time = 0. prof_pass = '******' except Exception: prof_time = 0 prof_id = '#' + str(test_id) prof_test = ('FAILED PARSING, see raw log for details' ' on test') prof_pass = '' prof_tuple = (prof_time, prof_id, prof_test, prof_pass) # appending tuple to master list prof_master_nosort.append(prof_tuple) # write the no sort file s_nosort = ((str(prof_tuple[0]) + 's').ljust(10) + " " + prof_tuple[1].ljust(7) + " " + prof_tuple[2] + prof_tuple[3] + "\n") f_nosort.write(s_nosort) f_nosort.flush() print('%s%% time-profiled' % ((test_id * 100) // n_tests)) f_rawlog.close() # sorting tests according to running-time prof_master_sort = sorted(prof_master_nosort, key=lambda test: test[0], reverse=True) # saving results to readable files path_sort = os.path.join(sav_dir, 'timeprof_sort') with open(path_sort, 'w') as f_sort: f_sort.write('TIME-PROFILING OF THEANO\'S NOSETESTS' ' (sorted by computation time)\n\n' + stamp + fields) for i in xrange(len(prof_master_nosort)): s_sort = ((str(prof_master_sort[i][0]) + 's').ljust(10) + " " + prof_master_sort[i][1].ljust(7) + " " + prof_master_sort[i][2] + prof_master_sort[i][3] + "\n") f_sort.write(s_sort)
import socket import subprocess import sys import textwrap import numpy import theano from theano.configparser import config, AddConfigVar, ConfigParam, StrParam from theano.gof.utils import flatten from theano.misc.windows import output_subprocess_Popen _logger = logging.getLogger("theano.gof.compiledir") try: p_out = output_subprocess_Popen([theano.config.cxx, '-dumpversion']) gcc_version_str = p_out[0].strip().decode() except OSError: # Typically means gcc cannot be found. gcc_version_str = 'GCC_NOT_FOUND' def local_bitwidth(): """ Return 32 for 32bit arch, 64 for 64bit arch By "architecture", we mean the size of memory pointers (size_t in C), *not* the size of long int, as it can be different. """ # Note that according to Python documentation, `platform.architecture()` is # not reliable on OS X with universal binaries.
def run(stdout, stderr, argv, theano_nose, batch_size, time_profile, display_batch_output): # Setting aside current working directory for later saving sav_dir = os.getcwd() # The first argument is the called script. argv = argv[1:] # It seems safer to fully regenerate the list of tests on each call. if os.path.isfile(".noseids"): os.remove(".noseids") # Collect test IDs. print( """\ #################### # COLLECTING TESTS # ####################""" ) stdout.flush() stderr.flush() dummy_in = open(os.devnull) # We need to call 'python' on Windows, because theano-nose is not a # native Windows app; and it does not hurt to call it on Unix. # Using sys.executable, so that the same Python version is used. python = sys.executable rval = subprocess.call( ([python, theano_nose, "--collect-only", "--with-id"] + argv), stdin=dummy_in.fileno(), stdout=stdout.fileno(), stderr=stderr.fileno(), ) stdout.flush() stderr.flush() assert rval == 0 noseids_file = ".noseids" with open(noseids_file, "rb") as f: data = cPickle.load(f) ids = data["ids"] n_tests = len(ids) if n_tests == 0: raise Exception("0 test selected") assert n_tests == max(ids) # Standard batch testing is called for if not time_profile: failed = set() print( """\ ################################### # RUNNING TESTS IN BATCHES OF %s # ###################################""" % batch_size ) # When `display_batch_output` is False, we suppress all output because # we want the user to focus only on the failed tests, which are re-run # (with output) below. dummy_out = open(os.devnull, "w") for test_id in xrange(1, n_tests + 1, batch_size): stdout.flush() stderr.flush() test_range = range(test_id, min(test_id + batch_size, n_tests + 1)) cmd = [python, theano_nose, "--with-id"] + map(str, test_range) + argv subprocess_extra_args = dict(stdin=dummy_in.fileno()) if not display_batch_output: # Use quiet mode in nosetests. cmd.append("-q") # Suppress all output. subprocess_extra_args.update(dict(stdout=dummy_out.fileno(), stderr=dummy_out.fileno())) t0 = time.time() subprocess.call(cmd, **subprocess_extra_args) t1 = time.time() # Recover failed test indices from the 'failed' field of the # '.noseids' file. We need to do it after each batch because # otherwise this field may get erased. We use a set because it # seems like it is not systematically erased though, and we want # to avoid duplicates. with open(noseids_file, "rb") as f: failed = failed.union(cPickle.load(f)["failed"]) print("%s%% done in %.3fs (failed: %s)" % ((test_range[-1] * 100) // n_tests, t1 - t0, len(failed))) # Sort for cosmetic purpose only. failed = sorted(failed) if failed: # Re-run only failed tests print( """\ ################################ # RE-RUNNING FAILED TESTS ONLY # ################################""" ) stdout.flush() stderr.flush() subprocess.call( ([python, theano_nose, "-v", "--with-id"] + failed + argv), stdin=dummy_in.fileno(), stdout=stdout.fileno(), stderr=stderr.fileno(), ) stdout.flush() stderr.flush() return 0 else: print( """\ #################### # ALL TESTS PASSED # ####################""" ) # Time-profiling is called for else: print( """\ ######################################## # RUNNING TESTS IN TIME-PROFILING MODE # ########################################""" ) # finds first word of list l containing string s def getIndexOfFirst(l, s): for pos, word in enumerate(l): if s in word: return pos # finds last word of list l containing string s def getIndexOfLast(l, s): for pos, word in enumerate(reversed(l)): if s in word: return len(l) - pos - 1 # iterating through tests # initializing master profiling list and raw log prof_master_nosort = [] prof_rawlog = [] dummy_out = open(os.devnull, "w") path_rawlog = os.path.join(sav_dir, "timeprof_rawlog") stamp = str(datetime.datetime.now()) + "\n\n" f_rawlog = open(path_rawlog, "w") f_rawlog.write("TIME-PROFILING OF THEANO'S NOSETESTS" " (raw log)\n\n" + stamp) f_rawlog.flush() stamp = str(datetime.datetime.now()) + "\n\n" fields = "Fields: computation time; nosetests sequential id;" " test name; parent class (if any); outcome\n\n" path_nosort = os.path.join(sav_dir, "timeprof_nosort") f_nosort = open(path_nosort, "w") f_nosort.write("TIME-PROFILING OF THEANO'S NOSETESTS" " (by sequential id)\n\n" + stamp + fields) f_nosort.flush() for test_floor in xrange(1, n_tests + 1, batch_size): for test_id in xrange(test_floor, min(test_floor + batch_size, n_tests + 1)): # Print the test we will start in the raw log to help # debug tests that are too long. f_rawlog.write("\n%s Will run test #%d %s\n" % (time.ctime(), test_id, data["ids"][test_id])) f_rawlog.flush() p_out = output_subprocess_Popen( ([python, theano_nose, "-v", "--with-id"] + [str(test_id)] + argv + ["--disabdocstring"]) ) # the previous option calls a custom Nosetests plugin # precluding automatic sustitution of doc. string for # test name in display # (see class 'DisabDocString' in file theano-nose) # recovering and processing data from pipe err = p_out[1] # print the raw log f_rawlog.write(err) f_rawlog.flush() # parsing the output l_err = err.split() try: pos_id = getIndexOfFirst(l_err, "#") prof_id = l_err[pos_id] pos_dot = getIndexOfFirst(l_err, "...") prof_test = "" for s in l_err[pos_id + 1 : pos_dot]: prof_test += s + " " if "OK" in err: pos_ok = getIndexOfLast(l_err, "OK") if len(l_err) == pos_ok + 1: prof_time = float(l_err[pos_ok - 1][0:-1]) prof_pass = "******" elif "SKIP" in l_err[pos_ok + 1]: prof_time = 0.0 prof_pass = "******" elif "KNOWNFAIL" in l_err[pos_ok + 1]: prof_time = float(l_err[pos_ok - 1][0:-1]) prof_pass = "******" else: prof_time = 0.0 prof_pass = "******" else: prof_time = 0.0 prof_pass = "******" except Exception: prof_time = 0 prof_id = "#" + str(test_id) prof_test = "FAILED PARSING, see raw log for details" " on test" prof_pass = "" prof_tuple = (prof_time, prof_id, prof_test, prof_pass) # appending tuple to master list prof_master_nosort.append(prof_tuple) # write the no sort file s_nosort = ( (str(prof_tuple[0]) + "s").ljust(10) + " " + prof_tuple[1].ljust(7) + " " + prof_tuple[2] + prof_tuple[3] + "\n" ) f_nosort.write(s_nosort) f_nosort.flush() print("%s%% time-profiled" % ((test_id * 100) // n_tests)) f_rawlog.close() # sorting tests according to running-time prof_master_sort = sorted(prof_master_nosort, key=lambda test: test[0], reverse=True) # saving results to readable files path_sort = os.path.join(sav_dir, "timeprof_sort") f_sort = open(path_sort, "w") f_sort.write("TIME-PROFILING OF THEANO'S NOSETESTS" " (sorted by computation time)\n\n" + stamp + fields) for i in xrange(len(prof_master_nosort)): s_sort = ( (str(prof_master_sort[i][0]) + "s").ljust(10) + " " + prof_master_sort[i][1].ljust(7) + " " + prof_master_sort[i][2] + prof_master_sort[i][3] + "\n" ) f_sort.write(s_sort) f_nosort.close() f_sort.close()
import subprocess import sys import textwrap import numpy import theano from theano.configparser import config, AddConfigVar, ConfigParam, StrParam from theano.gof.utils import flatten from theano.misc.windows import output_subprocess_Popen _logger = logging.getLogger("theano.gof.compiledir") try: p_out = output_subprocess_Popen([theano.config.cxx, '-dumpversion']) gcc_version_str = p_out[0].strip().decode() except OSError: # Typically means gcc cannot be found. gcc_version_str = 'GCC_NOT_FOUND' def local_bitwidth(): """ Return 32 for 32bit arch, 64 for 64bit arch By "architecture", we mean the size of memory pointers (size_t in C), *not* the size of long int, as it can be different. """ # Note that according to Python documentation, `platform.architecture()` is # not reliable on OS X with universal binaries.
import subprocess import sys import textwrap import numpy import theano from theano.configparser import config, AddConfigVar, ConfigParam, StrParam from theano.gof.utils import flatten from theano.misc.windows import output_subprocess_Popen _logger = logging.getLogger("theano.gof.compiledir") try: p_out = output_subprocess_Popen(['g++', '-dumpversion']) gcc_version_str = p_out[0].strip().decode() except OSError: # Typically means gcc cannot be found. gcc_version_str = 'GCC_NOT_FOUND' def local_bitwidth(): """ Return 32 for 32bit arch, 64 for 64bit arch By "architecture", we mean the size of memory pointers (size_t in C), *not* the size of long int, as it can be different. """ # Note that according to Python documentation, `platform.architecture()` is # not reliable on OS X with universal binaries.
import socket import subprocess import sys import textwrap import numpy import theano from theano.configparser import config, AddConfigVar, ConfigParam, StrParam from theano.gof.utils import flatten from theano.misc.windows import output_subprocess_Popen _logger = logging.getLogger("theano.gof.compiledir") try: p_out = output_subprocess_Popen(['g++', '-dumpversion']) gcc_version_str = p_out[0].strip().decode() except OSError: # Typically means gcc cannot be found. gcc_version_str = 'GCC_NOT_FOUND' def local_bitwidth(): """ Return 32 for 32bit arch, 64 for 64bit arch By "architecture", we mean the size of memory pointers (size_t in C), *not* the size of long int, as it can be different. """ # Note that according to Python documentation, `platform.architecture()` is # not reliable on OS X with universal binaries.