def test_nt_quote_args(self): for args, wanted in ((['with space', 'nospace'], ['"with space"', 'nospace']), (['nochange', 'nospace'], ['nochange', 'nospace'])): res = _nt_quote_args(args) self.assertEqual(res, wanted)
def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): """Compile 'src' to product 'obj'.""" if is_f_file(src) and not has_f90_header(src): flavor = ':f77' compiler = self.compiler_f77 elif is_free_format(src): flavor = ':f90' compiler = self.compiler_f90 if compiler is None: raise DistutilsExecError, 'f90 not supported by '\ +self.__class__.__name__ else: flavor = ':fix' compiler = self.compiler_fix if compiler is None: raise DistutilsExecError, 'f90 (fixed) not supported by '\ +self.__class__.__name__ if self.object_switch[-1]==' ': o_args = [self.object_switch.strip(),obj] else: o_args = [self.object_switch.strip()+obj] assert self.compile_switch.strip() s_args = [self.compile_switch, src] if os.name == 'nt': compiler = _nt_quote_args(compiler) command = compiler + cc_args + s_args + o_args + extra_postargs display = '%s: %s' % (os.path.basename(compiler[0]) + flavor, src) try: self.spawn(command,display=display) except DistutilsExecError, msg: raise CompileError, msg
def spawn(cmd): # Using regex to match src and obj src_regex = re.compile('/T(p|c)(.*)') src_list = [m.group(2) for m in ( src_regex.match(elem) for elem in cmd) if m] obj_regex = re.compile('/Fo(.*)') obj_list = [m.group(1) for m in ( obj_regex.match(elem) for elem in cmd) if m] if len(src_list) >= 1 and len(obj_list) >= 1: src = src_list[0] obj = obj_list[0] else: # Cannot find src or obj, revert back to original style return orig_spawn(cmd) quote_regex = re.compile('".*"') quote_list = [quote_regex.search( arg) is not None for arg in cmd] no_quote = any(quote_list) if not no_quote: from distutils.spawn import _nt_quote_args cmd = _nt_quote_args(cmd) builder.writer.build( [obj], 'compile', [src], variables={ 'cmd': cmd, 'deps': 'msvc' })
def test_nt_quote_args(self): for (args, wanted) in ((['with space', 'nospace'], ['"with space"', 'nospace']), (['nochange', 'nospace'], ['nochange', 'nospace'])): res = _nt_quote_args(args) self.assertEqual(res, wanted)
def spawn(self, cmd, display=None): if type(cmd) is type([]) and os.name == 'nt': cmd = _nt_quote_args(cmd) s,o = exec_command(cmd, use_tee=0) if s: from distutils.ccompiler import DistutilsExecError raise DistutilsExecError,\ 'Command "%s" failed with exit status %d' % (cmd, s)
def test_nt_quote_args(self): for (args, wanted) in ( (["with space", "nospace"], ['"with space"', "nospace"]), (["nochange", "nospace"], ["nochange", "nospace"]), ): res = _nt_quote_args(args) self.assertEqual(res, wanted)
def CCompiler_spawn(self, cmd, display=None): if display is None: display = cmd if type(display) is type([]): display = ' '.join(display) log.info(display) if type(cmd) is type([]) and os.name == 'nt': cmd = _nt_quote_args(cmd) s,o = exec_command(cmd) if s: if type(cmd) is type([]): cmd = ' '.join(cmd) print o raise DistutilsExecError,\ 'Command "%s" failed with exit status %d' % (cmd, s)
def CCompiler_spawn(self, cmd, display=None): if display is None: display = cmd if is_sequence(display): display = ' '.join(list(display)) log.info(display) if is_sequence(cmd) and os.name == 'nt': cmd = _nt_quote_args(list(cmd)) s,o = exec_command(cmd) if s: if is_sequence(cmd): cmd = ' '.join(list(cmd)) print o raise DistutilsExecError,\ 'Command "%s" failed with exit status %d' % (cmd, s)
def link(self, target_desc, objects, output_filename, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None): objects, output_dir = self._fix_object_args(objects, output_dir) libraries, library_dirs, runtime_library_dirs = \ self._fix_lib_args(libraries, library_dirs, runtime_library_dirs) lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, libraries) if type(output_dir) not in (StringType, NoneType): raise TypeError, "'output_dir' must be a string or None" if output_dir is not None: output_filename = os.path.join(output_dir, output_filename) if self._need_link(objects, output_filename): if self.library_switch[-1]==' ': o_args = [self.library_switch.strip(),output_filename] else: o_args = [self.library_switch.strip()+output_filename] if type(self.objects) is type(''): ld_args = objects + [self.objects] else: ld_args = objects + self.objects ld_args = ld_args + lib_opts + o_args if debug: ld_args[:0] = ['-g'] if extra_preargs: ld_args[:0] = extra_preargs if extra_postargs: ld_args.extend(extra_postargs) self.mkpath(os.path.dirname(output_filename)) if target_desc == CCompiler.EXECUTABLE: raise NotImplementedError,self.__class__.__name__+'.linker_exe attribute' else: linker = self.linker_so[:] if os.name == 'nt': linker = _nt_quote_args(linker) command = linker + ld_args try: self.spawn(command) except DistutilsExecError, msg: raise LinkError, msg
def link(self, target_desc, objects, output_filename, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None): objects, output_dir = self._fix_object_args(objects, output_dir) libraries, library_dirs, runtime_library_dirs = \ self._fix_lib_args(libraries, library_dirs, runtime_library_dirs) lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, libraries) if is_string(output_dir): output_filename = os.path.join(output_dir, output_filename) elif output_dir is not None: raise TypeError, "'output_dir' must be a string or None" if self._need_link(objects, output_filename): if self.library_switch[-1]==' ': o_args = [self.library_switch.strip(),output_filename] else: o_args = [self.library_switch.strip()+output_filename] if is_string(self.objects): ld_args = objects + [self.objects] else: ld_args = objects + self.objects ld_args = ld_args + lib_opts + o_args if debug: ld_args[:0] = ['-g'] if extra_preargs: ld_args[:0] = extra_preargs if extra_postargs: ld_args.extend(extra_postargs) self.mkpath(os.path.dirname(output_filename)) if target_desc == CCompiler.EXECUTABLE: linker = self.linker_exe[:] else: linker = self.linker_so[:] if os.name == 'nt': linker = _nt_quote_args(linker) command = linker + ld_args try: self.spawn(command) except DistutilsExecError, msg: raise LinkError, msg
def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): """Compile 'src' to product 'obj'.""" src_flags = {} if is_f_file(src) and not has_f90_header(src): flavor = ':f77' compiler = self.compiler_f77 src_flags = get_f77flags(src) elif is_free_format(src): flavor = ':f90' compiler = self.compiler_f90 if compiler is None: raise DistutilsExecError, 'f90 not supported by %s needed for %s'\ % (self.__class__.__name__,src) else: flavor = ':fix' compiler = self.compiler_fix if compiler is None: raise DistutilsExecError, 'f90 (fixed) not supported by %s needed for %s'\ % (self.__class__.__name__,src) if self.object_switch[-1]==' ': o_args = [self.object_switch.strip(),obj] else: o_args = [self.object_switch.strip()+obj] assert self.compile_switch.strip() s_args = [self.compile_switch, src] extra_flags = src_flags.get(self.compiler_type,[]) if extra_flags: log.info('using compile options from source: %r' \ % ' '.join(extra_flags)) if os.name == 'nt': compiler = _nt_quote_args(compiler) command = compiler + cc_args + extra_flags + s_args + o_args + extra_postargs display = '%s: %s' % (os.path.basename(compiler[0]) + flavor, src) try: self.spawn(command,display=display) except DistutilsExecError, msg: raise CompileError, msg
def _write_ninja_file(path, name, sources, extra_cflags, extra_cuda_cflags, extra_ldflags, extra_include_paths, with_cuda=False): # Version 1.3 is required for the `deps` directive. config = ['ninja_required_version = 1.3'] config.append('cxx = {}'.format(os.environ.get('CXX', 'c++'))) if with_cuda: config.append('nvcc = {}'.format(_join_cuda_home('bin', 'nvcc'))) # Turn into absolute paths so we can emit them into the ninja build # file wherever it is. sources = [os.path.abspath(file) for file in sources] includes = [os.path.abspath(file) for file in extra_include_paths] # include_paths() gives us the location of torch/python.h includes += include_paths(with_cuda) # sysconfig.get_paths()['include'] gives us the location of Python.h includes.append(sysconfig.get_paths()['include']) common_cflags = ['-DTORCH_EXTENSION_NAME={}'.format(name)] common_cflags += ['-I{}'.format(include) for include in includes] cflags = common_cflags + ['-fPIC', '-std=c++11'] + extra_cflags if sys.platform == 'win32': from distutils.spawn import _nt_quote_args cflags = _nt_quote_args(cflags) flags = ['cflags = {}'.format(' '.join(cflags))] if with_cuda: cuda_flags = common_cflags if sys.platform == 'win32': cuda_flags = _nt_quote_args(cuda_flags) else: cuda_flags += ['--compiler-options', "'-fPIC'"] cuda_flags += extra_cuda_cflags if not any(flag.startswith('-std=') for flag in cuda_flags): cuda_flags.append('-std=c++11') flags.append('cuda_flags = {}'.format(' '.join(cuda_flags))) if sys.platform == 'win32': ldflags = ['/DLL'] + extra_ldflags else: ldflags = ['-shared'] + extra_ldflags # The darwin linker needs explicit consent to ignore unresolved symbols. if sys.platform == 'darwin': ldflags.append('-undefined dynamic_lookup') elif sys.platform == 'win32': ldflags = _nt_quote_args(ldflags) flags.append('ldflags = {}'.format(' '.join(ldflags))) # See https://ninja-build.org/build.ninja.html for reference. compile_rule = ['rule compile'] if sys.platform == 'win32': compile_rule.append( ' command = cl /showIncludes $cflags -c $in /Fo$out') compile_rule.append(' deps = msvc') else: compile_rule.append( ' command = $cxx -MMD -MF $out.d $cflags -c $in -o $out') compile_rule.append(' depfile = $out.d') compile_rule.append(' deps = gcc') if with_cuda: cuda_compile_rule = ['rule cuda_compile'] cuda_compile_rule.append( ' command = $nvcc $cuda_flags -c $in -o $out') link_rule = ['rule link'] if sys.platform == 'win32': cl_paths = subprocess.check_output(['where', 'cl']).decode().split('\r\n') if len(cl_paths) >= 1: cl_path = os.path.dirname(cl_paths[0]).replace(':', '$:') else: raise RuntimeError("MSVC is required to load C++ extensions") link_rule.append( ' command = "{}/link.exe" $in /nologo $ldflags /out:$out'.format( cl_path)) else: link_rule.append(' command = $cxx $ldflags $in -o $out') # Emit one build rule per source to enable incremental build. object_files = [] build = [] for source_file in sources: # '/path/to/file.cpp' -> 'file' file_name = os.path.splitext(os.path.basename(source_file))[0] if _is_cuda_file(source_file): rule = 'cuda_compile' # Use a different object filename in case a C++ and CUDA file have # the same filename but different extension (.cpp vs. .cu). target = '{}.cuda.o'.format(file_name) else: rule = 'compile' target = '{}.o'.format(file_name) object_files.append(target) if sys.platform == 'win32': source_file = source_file.replace(':', '$:') build.append('build {}: {} {}'.format(target, rule, source_file)) ext = '.pyd' if sys.platform == 'win32' else '.so' library_target = '{}{}'.format(name, ext) link = ['build {}: link {}'.format(library_target, ' '.join(object_files))] default = ['default {}'.format(library_target)] # 'Blocks' should be separated by newlines, for visual benefit. blocks = [config, flags, compile_rule] if with_cuda: blocks.append(cuda_compile_rule) blocks += [link_rule, build, link, default] with open(path, 'w') as build_file: for block in blocks: lines = '\n'.join(block) build_file.write('{}\n\n'.format(lines))
def _write_ninja_file(path, name, sources, extra_cflags, extra_cuda_cflags, extra_ldflags, extra_include_paths, with_cuda=False): extra_cflags = [flag.strip() for flag in extra_cflags] extra_cuda_cflags = [flag.strip() for flag in extra_cuda_cflags] extra_ldflags = [flag.strip() for flag in extra_ldflags] extra_include_paths = [flag.strip() for flag in extra_include_paths] # Version 1.3 is required for the `deps` directive. config = ['ninja_required_version = 1.3'] config.append('cxx = {}'.format(os.environ.get('CXX', 'c++'))) if with_cuda: config.append('nvcc = {}'.format(_join_cuda_home('bin', 'nvcc'))) # Turn into absolute paths so we can emit them into the ninja build # file wherever it is. sources = [os.path.abspath(file) for file in sources] includes = [os.path.abspath(file) for file in extra_include_paths] # include_paths() gives us the location of torch/torch.h includes += include_paths(with_cuda) # sysconfig.get_paths()['include'] gives us the location of Python.h includes.append(sysconfig.get_paths()['include']) common_cflags = ['-DTORCH_EXTENSION_NAME={}'.format(name)] common_cflags += ['-I{}'.format(include) for include in includes] cflags = common_cflags + ['-fPIC', '-std=c++11'] + extra_cflags if sys.platform == 'win32': from distutils.spawn import _nt_quote_args cflags = _nt_quote_args(cflags) flags = ['cflags = {}'.format(' '.join(cflags))] if with_cuda: cuda_flags = common_cflags if sys.platform == 'win32': cuda_flags = _nt_quote_args(cuda_flags) else: cuda_flags += ['--compiler-options', "'-fPIC'"] cuda_flags += extra_cuda_cflags if not any(flag.startswith('-std=') for flag in cuda_flags): cuda_flags.append('-std=c++11') flags.append('cuda_flags = {}'.format(' '.join(cuda_flags))) if sys.platform == 'win32': ldflags = ['/DLL'] + extra_ldflags else: ldflags = ['-shared'] + extra_ldflags # The darwin linker needs explicit consent to ignore unresolved symbols. if sys.platform == 'darwin': ldflags.append('-undefined dynamic_lookup') elif sys.platform == 'win32': ldflags = _nt_quote_args(ldflags) flags.append('ldflags = {}'.format(' '.join(ldflags))) # See https://ninja-build.org/build.ninja.html for reference. compile_rule = ['rule compile'] if sys.platform == 'win32': compile_rule.append( ' command = cl /showIncludes $cflags -c $in /Fo$out') compile_rule.append(' deps = msvc') else: compile_rule.append( ' command = $cxx -MMD -MF $out.d $cflags -c $in -o $out') compile_rule.append(' depfile = $out.d') compile_rule.append(' deps = gcc') if with_cuda: cuda_compile_rule = ['rule cuda_compile'] cuda_compile_rule.append( ' command = $nvcc $cuda_flags -c $in -o $out') link_rule = ['rule link'] if sys.platform == 'win32': cl_paths = subprocess.check_output(['where', 'cl']).decode().split('\r\n') if len(cl_paths) >= 1: cl_path = os.path.dirname(cl_paths[0]).replace(':', '$:') else: raise RuntimeError("MSVC is required to load C++ extensions") link_rule.append( ' command = "{}/link.exe" $in /nologo $ldflags /out:$out'.format( cl_path)) else: link_rule.append(' command = $cxx $ldflags $in -o $out') # Emit one build rule per source to enable incremental build. object_files = [] build = [] for source_file in sources: # '/path/to/file.cpp' -> 'file' file_name = os.path.splitext(os.path.basename(source_file))[0] if _is_cuda_file(source_file) and with_cuda: rule = 'cuda_compile' # Use a different object filename in case a C++ and CUDA file have # the same filename but different extension (.cpp vs. .cu). target = '{}.cuda.o'.format(file_name) else: rule = 'compile' target = '{}.o'.format(file_name) object_files.append(target) if sys.platform == 'win32': source_file = source_file.replace(':', '$:') build.append('build {}: {} {}'.format(target, rule, source_file)) ext = '.pyd' if sys.platform == 'win32' else '.so' library_target = '{}{}'.format(name, ext) link = ['build {}: link {}'.format(library_target, ' '.join(object_files))] default = ['default {}'.format(library_target)] # 'Blocks' should be separated by newlines, for visual benefit. blocks = [config, flags, compile_rule] if with_cuda: blocks.append(cuda_compile_rule) blocks += [link_rule, build, link, default] with open(path, 'w') as build_file: for block in blocks: lines = '\n'.join(block) build_file.write('{}\n\n'.format(lines))