def extract(self): os.chmod(self.runfile, 0o777) with tempdir() as tmpd: if self.embedded_blob is not None: with tempdir() as tmpd2: cmd = [ os.path.join(self.src_dir, self.runfile), "--extract=%s" % (tmpd2, ), "--nox11", "--silent", ] check_call(cmd) # extract the embedded blob cmd = [ os.path.join(tmpd2, self.embedded_blob), "-prefix", tmpd, "-noprompt", "--nox11", ] check_call(cmd) else: # Current Nvidia's Linux based runfiles don't use embedded runfiles # # "--installpath" runfile command is used to install the toolkit to a specified # directory with the contents and layout similar to an install to # '/usr/local/cuda` # "--override" runfile command to disable the compiler check since we are not # installing the driver here # "--nox11" runfile command prevents desktop GUI on local install cmd = [ os.path.join(self.src_dir, self.runfile), "--installpath=%s" % (tmpd), "--toolkit", "--silent", "--override", "--nox11", ] check_call(cmd) for p in self.patches: os.chmod(p, 0o777) cmd = [ os.path.join(self.src_dir, p), "--installdir", tmpd, "--accept-eula", "--silent", ] check_call(cmd) self.copy(tmpd)
def copy(source, dest): isodir = tempfile.tempdir() banner("Copying %s to local disk" % source) mount(source, isodir) subprocess.call(['mkdir', '-p', dest]) subprocess.call(['cp', '-r', isodir, dest]) umount(isodir)
def extract(self): runfile = self.cu_blob patches = self.patches os.chmod(runfile, 0o777) with tempdir() as tmpd: if "10.1" in self.cu_version: cmd = [ os.path.join(self.src_dir, runfile), '--installpath=' + tmpd, '--toolkit', '--silent', '--override' ] else: cmd = [ os.path.join(self.src_dir, runfile), '--toolkitpath', tmpd, '--toolkit', '--silent' ] check_call(cmd) for p in patches: os.chmod(p, 0o777) if "10.1" in self.cu_version: cmd = [ os.path.join(self.src_dir, runfile), '--installdir=' + tmpd, '--accept-eula', '--silent' ] else: cmd = [ os.path.join(self.src_dir, p), '--installdir', tmpd, '--accept-eula', '--silent' ] check_call(cmd) self.copy(tmpd)
def extract(self): print("Extracting on Linux") runfile = self.blob_dir / self.cu_blob os.chmod(runfile, 0o777) with tempdir() as tmpdir: cmd = [ str(runfile), "--silent", "--toolkit", f"--toolkitpath={tmpdir}", "--override" ] subprocess.run(cmd, env=os.environ.copy(), check=True) toolkitpath = tmpdir if not os.path.isdir(toolkitpath): print('STATUS:', status) for fn in glob.glob('/tmp/cuda_install_*.log'): f = open(fn, 'r') print('-' * 100, fn) print(f.read()) print('-' * 100) f.close() os.system('ldd --version') os.system('ls -la %s' % (tmpdir)) raise RuntimeError( 'Something went wrong in executing `{}`: directory `{}` does not exists' .format(' '.join(cmd), toolkitpath)) self.copy_files(toolkitpath, self.src_dir) os.remove(runfile)
def extract(self): runfile = self.cu_blob patches = self.patches try: with tempdir() as tmpd: extract_name = '__extracted' extractdir = os.path.join(tmpd, extract_name) os.mkdir(extract_name) check_call([ '7za', 'x', '-o%s' % extractdir, os.path.join(self.src_dir, runfile) ]) for p in patches: check_call([ '7za', 'x', '-aoa', '-o%s' % extractdir, os.path.join(self.src_dir, p) ]) nvt_path = os.environ.get('NVTOOLSEXT_INSTALL_PATH', self.nvtoolsextpath) print("NvToolsExt path: %s" % nvt_path) if nvt_path is not None: if not Path(nvt_path).is_dir(): msg = ("NVTOOLSEXT_INSTALL_PATH is invalid " "or inaccessible.") raise ValueError(msg) # fetch all the dlls into DLLs store_name = 'DLLs' store = os.path.join(tmpd, store_name) os.mkdir(store) for path, dirs, files in os.walk(extractdir): if 'jre' not in path: # don't get jre dlls for filename in fnmatch.filter(files, "*.dll"): if not Path(os.path.join(store, filename)).is_file(): shutil.copy(os.path.join(path, filename), store) for filename in fnmatch.filter(files, "*.bc"): if not Path(os.path.join(store, filename)).is_file(): shutil.copy(os.path.join(path, filename), store) if nvt_path is not None: for path, dirs, files in os.walk(nvt_path): for filename in fnmatch.filter(files, "*.dll"): if not Path(os.path.join(store, filename)).is_file(): shutil.copy(os.path.join(path, filename), store) self.copy(store) except PermissionError: # TODO: fix this # cuda 8 has files that refuse to delete, figure out perm changes # needed and apply them above, tempdir context exit fails to rmtree pass
def _mount_extract(self, image, store): with tempdir() as tmpmnt: with _hdiutil_mount(tmpmnt, os.path.join(os.getcwd(), image)) as mntpnt: for tlpath, tldirs, tlfiles in os.walk(mntpnt): for tzfile in fnmatch.filter(tlfiles, "*.tar.gz"): with tarfile.open(os.path.join(tlpath, tzfile)) as tar: tar.extractall(store, members=self._extract_matcher(tar))
def get_data(self): try: path = self.config['mmstats']['mmstats_path'] except KeyError: path = tempdir() GLOB = os.path.join(path, 'mmstats-*') stats = {} for _, label, value in iter_stats(GLOB): stats.update({label: value}) return stats
def extract(self): runfile = self.config_blob patches = self.patches os.chmod(runfile, 0o777) with tempdir() as tmpd: if self.embedded_blob is not None: with tempdir() as tmpd2: cmd = [ os.path.join(self.src_dir, runfile), '--extract=%s' % (tmpd2, ), '--nox11', '--silent' ] check_call(cmd) # extract the embedded blob cmd = [ os.path.join(tmpd2, self.embedded_blob), '-prefix', tmpd, '-noprompt', '--nox11' ] check_call(cmd) else: # Nvidia's RHEL7 based runfiles don't use embedded runfiles # Once the toolkit is extracted, it ends up in a directory called "cuda-toolkit' # the --extract runfile command is used because letting the runfile do an "install" # results in attempted installs of .pc and doc files into standard Linux locations, # which is not what we want. # The "--override" runfile command to disable the compiler check since we are not # installing the driver here. cmd = [ os.path.join(self.src_dir, runfile), '--extract=%s' % (tmpd), '--toolkit', '--silent', '--override' ] check_call(cmd) for p in patches: os.chmod(p, 0o777) cmd = [ os.path.join(self.src_dir, p), '--installdir', tmpd, '--accept-eula', '--silent' ] check_call(cmd) self.copy(tmpd)
def extract(self): runfile = self.config_blob patches = self.patches os.chmod(runfile, 0o777) with tempdir() as tmpd: if self.embedded_blob is not None: with tempdir() as tmpd2: cmd = [os.path.join(self.src_dir, runfile), '--extract=%s' % (tmpd2, ), '--nox11', '--silent'] check_call(cmd) # extract the embedded blob cmd = [os.path.join(tmpd2, self.embedded_blob), '-prefix', tmpd, '-noprompt', '--nox11'] check_call(cmd) else: cmd = [os.path.join(self.src_dir, runfile), '--toolkitpath', tmpd, '--toolkit', '--silent'] check_call(cmd) for p in patches: os.chmod(p, 0o777) cmd = [os.path.join(self.src_dir, p), '--installdir', tmpd, '--accept-eula', '--silent'] check_call(cmd) self.copy(tmpd)
def extract(self): runfile = self.cu_blob patches = self.patches os.chmod(runfile, 0o777) with tempdir() as tmpd: check_call([ os.path.join(self.src_dir, runfile), '--toolkitpath', tmpd, '--toolkit', '--silent' ]) for p in patches: os.chmod(p, 0o777) check_call([ os.path.join(self.src_dir, p), '--installdir', tmpd, '--accept-eula', '--silent' ]) self.copy(tmpd)
def extract(self): print("Extracting on Windows") runfile = self.blob_dir / self.cu_blob with tempdir() as tmpdir: cmd = ["7za", "x", str(runfile), f"-o{tmpdir}"] subprocess.run(cmd, env=os.environ.copy(), check=True) toolkitpath = tmpdir if not os.path.isdir(toolkitpath): print('STATUS:', status) os.system('dir %s' % (tmpdir)) raise RuntimeError( 'Something went wrong in executing `{}`: directory `{}` does not exist' .format(' '.join(cmd), toolkitpath)) # Copy installation to pkgs folder, to be linked to conda_prefix by hardlinks. # Hardlinks are selected over symlinks, because Windows 10 requires either admin privileges or developer mode enabled (since Creators Update) for the creation of symlinks. # These options are not guaranteed at the user end. target_dir = self.src_dir nvcc_dir = os.path.join(target_dir, "nvcc") # ignore=shutil.ignore_patterns('*.nvi') for toolkitpathroot, subdirs, files in os.walk(toolkitpath): for file in files: src_file = os.path.join(toolkitpathroot, file) os.chmod(src_file, 0o777) for subdir in subdirs: if subdir in [ 'CUDAVisualStudioIntegration' ] and (subdir not in Path(toolkitpathroot).parts): src = os.path.join(toolkitpathroot, subdir) dst = os.path.join(target_dir, subdir) copy_tree(src, dst) elif subdir in [ 'bin', 'include', 'lib', 'extras', 'libdevice', 'nvvm' ] and (subdir not in Path(toolkitpathroot).parts): src = os.path.join(toolkitpathroot, subdir) nvcc_dst = os.path.join(nvcc_dir, subdir) copy_tree(src, nvcc_dst) os.remove(runfile) # create hard links of the whole toolkit into %LIBRARY_PREFIX% self.create_hardlinks_into_prefix()
def extract(self): # Use dpkg to extract the libraries from a CUDA deb file with tempdir() as tmpd: extractdir = os.path.join(tmpd, "__extracted") # walk the extraction looking for more deb files embedded for path, dirs, files in os.walk(self.src_dir): for filename in files: if filename.lower().endswith('.deb'): cmd = [ 'dpkg', '-x', os.path.join(path, filename), extractdir ] check_call(cmd) self.copy(tmpd)
def extract(self): print("Extracting on Linux") runfile = self.blob_dir / self.cu_blob os.chmod(runfile, 0o777) with tempdir() as tmpdir: cmd = [ str(runfile), f"--extract={tmpdir}", #f"--defaultroot={tmpdir}", "--override" ] status = subprocess.run(cmd, check=True) toolkitpath = os.path.join(tmpdir, "cuda-toolkit") if not os.path.isdir(toolkitpath): installer = (glob.glob(os.path.join(tmpdir, 'cuda-linux*.run')) or [None])[0] if installer is not None: print('Try using', installer) subprocess.run( [ installer, '-prefix=%s' % (toolkitpath), '-noprompt' # Implies acceptance of the EULA ], check=True) if not os.path.isdir(toolkitpath): print('STATUS:', status) for fn in glob.glob('/tmp/cuda_install_*.log'): f = open(fn, 'r') print('-' * 100, fn) print(f.read()) print('-' * 100) f.close() os.system('ldd --version') os.system('ls -la %s' % (tmpdir)) raise RuntimeError( 'Something went wrong in executing `{}`: directory `{}` does not exists' .format(' '.join(cmd), toolkitpath)) self.copy_files(toolkitpath, self.src_dir) os.remove(runfile)
def extract(self): # For better error messages if os.path.exists("/tmp/cuda-installer.log"): try: os.remove("/tmp/cuda-installer.log") except OSError as e: raise RuntimeError( "Failed to remove /tmp/cuda-installer.log") from e print("Extracting on Linux") runfile = self.blob_dir / self.cu_blob os.chmod(runfile, 0o777) with tempdir() as tmpdir: cmd = [ str(runfile), "--silent", "--toolkit", f"--toolkitpath={tmpdir}", "--override" ] subprocess.run(cmd, env=os.environ.copy(), check=True) # Fix for conda-forge/cudatoolkit-dev-feedstock#44 if os.path.exists("/tmp/cuda-installer.log"): os.remove("/tmp/cuda-installer.log") toolkitpath = tmpdir if not os.path.isdir(toolkitpath): print('STATUS:', status) for fn in glob.glob('/tmp/cuda_install_*.log'): f = open(fn, 'r') print('-' * 100, fn) print(f.read()) print('-' * 100) f.close() os.system('ldd --version') os.system('ls -la %s' % (tmpdir)) raise RuntimeError( 'Something went wrong in executing `{}`: directory `{}` does not exist' .format(' '.join(cmd), toolkitpath)) self.copy_files(toolkitpath, self.src_dir) os.remove(runfile)
def extract(self): runfile = self.cu_blob patches = self.patches try: with tempdir() as tmpd: check_call([ '7za', 'x', '-o%s' % tmpd, os.path.join(self.src_dir, runfile) ]) for p in patches: check_call([ '7za', 'x', '-aoa', '-o%s' % tmpd, os.path.join(self.src_dir, p) ]) # fetch all the dlls into DLLs store_name = 'DLLs' store = os.path.join(tmpd, store_name) os.mkdir(store) for path, dirs, files in os.walk(tmpd): if 'jre' not in path: # don't get jre dlls for filename in fnmatch.filter(files, "*.dll"): if not Path(os.path.join(store, filename)).is_file(): shutil.copy(os.path.join(path, filename), store) for filename in fnmatch.filter(files, "*.bc"): if not Path(os.path.join(store, filename)).is_file(): shutil.copy(os.path.join(path, filename), store) self.copy(tmpd, store) except PermissionError: #TODO: fix this # cuda 8 has files that refuse to delete, figure out perm changes # needed and apply them above, tempdir context exit fails to rmtree pass
def extract(self): runfile = self.config_blob patches = self.patches with tempdir() as tmpd: # fetch all the dylibs into lib64, but first get them out of the # image and tar.gzs into tmpstore extract_store_name = 'tmpstore' extract_store = os.path.join(tmpd, extract_store_name) os.mkdir(extract_store) store_name = 'lib64' store = os.path.join(tmpd, store_name) os.mkdir(store) self._mount_extract(runfile, extract_store) for p in self.patches: self._mount_extract(p, extract_store) for path, dirs, files in os.walk(extract_store): for filename in fnmatch.filter(files, "*.dylib"): if not Path(os.path.join(store, filename)).is_file(): shutil.copy(os.path.join(path, filename), store) for filename in fnmatch.filter(files, "*.bc"): if not Path(os.path.join(store, filename)).is_file(): shutil.copy(os.path.join(path, filename), store) self.copy(tmpd, store)
def main(): # Get the source directory if len(sys.argv) == 2: path = sys.argv[1] else: path = '.' path = os.path.abspath(path) target = os.path.basename(path) + '.epub' temp = tempdir() title = os.path.basename(path) print('Source: %s' % path) print('Target: %s' % target) print(' Temp: %s' % temp) print() # Start an epub book = EpubBook() book.setTitle(title) book.addCreator('JP Verkamp') book.addTitlePage() book.addTocPage() # Run through the files and find the sections levels = [] sections = [] buffer = '' index = '1' header = '' min_depth = False for file in files(path): lines = skip_bom(open(file, 'r').read()).split('\n') for line in lines: if line.startswith('#'): if header: buffer = ''.join([c for c in buffer if ord(c) < 128]) sections.append((index, header, buffer)) buffer = line pounds, header = line.split(' ', 1) depth = pounds.count('#') if not min_depth or depth < min_depth: min_depth = depth if depth == len(levels): levels[-1] += 1 elif depth < len(levels): levels = levels[:depth] levels[-1] += 1 else: while depth > len(levels): levels.append(1) index = '.'.join([str(s) for s in levels]) else: buffer += line + '\n' if buffer: buffer = ''.join([c for c in buffer if ord(c) < 128]) sections.append((index, header, buffer)) # Generate the book contents i = 0 for index, header, content in sections: #if min_depth: # index = '.'.join(index.split('.')[min_depth:]) print('%s = %s' % (index, header)) i += 1 item = book.addHtml('', '%04d.html' % i, html(header, markdown(content))) book.addSpineItem(item) book.addTocMapNode(item.destPath, header, index.count('.') + 1) # Create the book book.createBook(temp) EpubBook.createArchive(temp, target) # Now try to create a mobi version os.system('ebook-convert "%s" "%s"' % (target, target.replace('epub', 'mobi')))
def start(self): self._array = [] self.index = 0 self._pics = tempfile.tempdir()
""" Convert all "*.txt" files in a folder from EBCDIC cp1141 to UTF-8. """ from __future__ import print_function from __future__ import unicode_literals import glob import os.path import tempfile # Register EBCDIC codecs. import ebcdic # noqa if __name__ == "__main__": source_encoding = "cp1141" target_encoding = "utf-8" print("convert from %s to %s:" % (source_encoding, target_encoding)) for source_path in glob.glob("*.txt"): target_name = os.path.splitext( os.path.basename(source_path))[0] + "_utf-8.txt" target_path = os.path.join(tempfile.tempdir(), target_name) print(" %s --> %s" % (source_path, target_path))
def __init__(self, flip=False, master_shader=False, gl_context=None, screenshots_dir=None): debug_prefix = "[MMVShaderMGL.__init__]" self.master_shader = master_shader # Screenshot dir only up to master shader if self.master_shader: # No given path if screenshots_dir is None: screenshots_dir = Path(tempfile.tempdir()) logging.info( f"{debug_prefix} Getting temp dir for screenshots") # Enforce pathlib.Path if isinstance(screenshots_dir, str): screenshots_dir = Path(screenshots_dir) # Assign self.screenshots_dir = screenshots_dir logging.info( f"{debug_prefix} Screenshots dir is [{self.screenshots_dir}]") # Name if self.master_shader: self.name = "Master Shader" else: self.name = str(uuid.uuid4()) # Config self.flip = flip self.ssaa = 1 # If you want to have some reset function to be called before reading shaders from path again # namely updating ShaderMaker self.__reset_function = self.dummy # Will be True if user presses "r" key, expected someone outside this class to revert to False # so we continue the cycle self.reset = False # We're a child shader so we inherit the parent (master shader) OpenGL context if gl_context is not None: self.gl_context = gl_context # Build coordinates based on self.flip self.__build_coordinates() # Info we send to the shaders read only (uniforms) that can and are generated on the main render loop self.pipeline = { # - mmv_time (float) -> time elapsed based on the frame rate of the shader in seconds "mmv_time": 0, # - mmv_frame (int) -> current frame number of the shader "mmv_frame": 0, # - mmv_resolution (vec2(int width, int height)) -> target output resolution "mmv_resolution": [0, 0], # - mmv_zoom (float) -> Scroll wheels zoom # - mmv_rotation (float) -> Target rotation in degrees "mmv_zoom": 1.0, "mmv_rotation": 0.0, # - mmv_drag (vec2(float)) -> Dragged pixels with mouse "mmv_drag": np.array([0.0, 0.0]), # Mouse position on screen, not normalized "mmv_mouse": [0, 0], # Are we flipped? "mmv_flip": -1 if self.flip else 1, } # If on real time view we want to freeze the shader for analysis self.freezed_pipeline = False # A dictionary that will hold indexes representing the location of textures # as well as the object we get it from: # # - Is it a static image? Do nothing just use its texture # - Is it a video? We need to get the next frame then upload to the GPU the texture # - Is it a shader? We need to render it # # This dictionary is mostly for information on which loader we have and to organize its # object if we need to do some action. self.textures = {} # Dictionary to hold name: indexes on self.textures dictionary for writing to textures # used for communicating big data arrays between Python and the GPU # One example usage is for writing FFT values self.writable_textures = {} # Initialize None values (self.width, self.height, self.fps) = [None] * 3 # # Refactors self.preprocessor = MMVShaderMGLPreprocessor(mmv_shader_mgl=self) self.window_handlers = MMVShaderMGLWindowHandlers(mmv_shader_mgl=self) # NOTE: Aliased functions on refactors, THIS MIGHT NOT APPEAR ON LINTING!! self.include_dir = self.preprocessor.include_dir self.mode = self.window_handlers.mode self.update_window = self.window_handlers.update_window
def get_test_tub_path(): tempdir()
def extract(self): os.chmod(self.runfile, 0o777) with tempdir() as tmpd: if self.embedded_blob is not None: with tempdir() as tmpd2: cmd = [ os.path.join(self.src_dir, self.runfile), "--extract=%s" % (tmpd2, ), "--nox11", "--silent", ] check_call(cmd) # extract the embedded blob cmd = [ os.path.join(tmpd2, self.embedded_blob), "-prefix", tmpd, "-noprompt", "--nox11", ] check_call(cmd) else: # Current Nvidia's Linux based runfiles don't use embedded runfiles # # "--installpath" runfile command is used to install the toolkit to a specified # directory with the contents and layout similar to an install to # '/usr/local/cuda` # "--override" runfile command to disable the compiler check since we are not # installing the driver here # "--nox11" runfile command prevents desktop GUI on local install cmd = [ os.path.join(self.src_dir, self.runfile), "--silent", "--override", "--nox11", "--toolkit", ] check = True # add toolkit install args if self.major_minor >= (10, 2): cmd.append(f"--installpath={tmpd}") elif self.major_minor == (10, 1): # v10.1 cmd.extend([ f"--toolkitpath={tmpd}", f"--librarypath={tmpd}", ]) if self.machine == "ppc64le": # cublas headers are not available, though the runfile # thinks that they are. check = False else: # <=10.0 cmd.append(f"--toolkitpath={tmpd}") self.run_extract(cmd, check=check) for p in self.patches: os.chmod(p, 0o777) cmd = [ os.path.join(self.src_dir, p), "--installdir", tmpd, "--accept-eula", "--silent", ] check_call(cmd) self.copy(tmpd)
def extract(self): try: with tempdir() as tmpd: extract_name = "__extracted" extractdir = os.path.join(tmpd, extract_name) os.mkdir(extract_name) check_call([ "7za", "x", "-o%s" % extractdir, os.path.join(self.src_dir, self.runfile), ]) for p in self.patches: check_call([ "7za", "x", "-aoa", "-o%s" % extractdir, os.path.join(self.src_dir, p), ]) nvt_path = os.environ.get("NVTOOLSEXT_INSTALL_PATH", self.nvtoolsextpath) print("NvToolsExt path: %s" % nvt_path) if nvt_path is not None: if not Path(nvt_path).is_dir(): msg = "NVTOOLSEXT_INSTALL_PATH is invalid " "or inaccessible." raise ValueError(msg) # fetch all the dlls into DLLs store_name = "DLLs" store = os.path.join(tmpd, store_name) os.mkdir(store) for path, dirs, files in os.walk(extractdir): if ("jre" not in path and "GFExperience" not in path): # don't get jre or GFExperience dlls for filename in fnmatch.filter(files, "*.dll"): if not Path(os.path.join(store, filename)).is_file(): shutil.copy(os.path.join(path, filename), store) for filename in fnmatch.filter(files, "*.lib"): if (path.endswith("x64") and not Path( os.path.join(store, filename)).is_file()): shutil.copy(os.path.join(path, filename), store) for filename in fnmatch.filter(files, "*.bc"): if not Path(os.path.join(store, filename)).is_file(): shutil.copy(os.path.join(path, filename), store) if nvt_path is not None: for path, dirs, files in os.walk(nvt_path): for filename in fnmatch.filter(files, "*.dll"): if not Path(os.path.join(store, filename)).is_file(): shutil.copy(os.path.join(path, filename), store) self.copy(store) except PermissionError: # TODO: fix this # cuda 8 has files that refuse to delete, figure out perm changes # needed and apply them above, tempdir context exit fails to rmtree pass