def create(cls, jobStore, leaderPath): """ Saves the content of the file or directory at the given path to the given job store and returns a resource object representing that content for the purpose of obtaining it again at a generic, public URL. This method should be invoked on the leader node. :rtype: Resource """ if os.path.isdir(leaderPath): if inVirtualEnv() and leaderPath.startswith(sys.prefix): subcls = VirtualEnvResource else: subcls = DirectoryResource elif os.path.isfile(leaderPath): subcls = FileResource elif os.path.exists(leaderPath): raise AssertionError("Neither a file or a directory: '%s'" % leaderPath) else: raise AssertionError("No such file or directory: '%s'" % leaderPath) pathHash = subcls._pathHash(leaderPath) contentHash = hashlib.md5() # noinspection PyProtectedMember with subcls._load(leaderPath) as src: with jobStore.writeSharedFileStream(pathHash, isProtected=False) as dst: userScript = src.read() contentHash.update(userScript) dst.write(userScript) return subcls(name=os.path.basename(leaderPath), pathHash=pathHash, url=(jobStore.getSharedPublicUrl(pathHash)), contentHash=contentHash.hexdigest())
def forModule(cls, name: str) -> Any: """ Return an instance of this class representing the module of the given name. If the given module name is "__main__", it will be translated to the actual file name of the top-level script without the .py or .pyc extension. This method assumes that the module with the specified name has already been loaded. """ module = sys.modules[name] filePath = os.path.abspath(module.__file__) filePath = filePath.split(os.path.sep) filePath[-1], extension = os.path.splitext(filePath[-1]) if extension not in (".py", ".pyc"): raise Exception( "The name of a user script/module must end in .py or .pyc.") if name == "__main__": logger.debug("Discovering real name of module") # User script/module was invoked as the main program if module.__package__: # Invoked as a module via python -m foo.bar logger.debug("Script was invoked as a module") name = [filePath.pop()] for package in reversed(module.__package__.split('.')): dirPathTail = filePath.pop() assert dirPathTail == package name.append(dirPathTail) name = '.'.join(reversed(name)) dirPath = os.path.sep.join(filePath) else: # Invoked as a script via python foo/bar.py name = filePath.pop() dirPath = os.path.sep.join(filePath) cls._check_conflict(dirPath, name) else: # User module was imported. Determine the directory containing the top-level package if filePath[-1] == '__init__': # module is a subpackage filePath.pop() for package in reversed(name.split('.')): dirPathTail = filePath.pop() assert dirPathTail == package dirPath = os.path.abspath(os.path.sep.join(filePath)) absPrefix = os.path.abspath(sys.prefix) inVenv = inVirtualEnv() logger.debug("Module dir is %s, our prefix is %s, virtualenv: %s", dirPath, absPrefix, inVenv) if not os.path.isdir(dirPath): raise Exception( f'Bad directory path {dirPath} for module {name}. Note that hot-deployment does not support .egg-link files yet, or scripts located in the root directory.' ) fromVirtualEnv = inVenv and dirPath.startswith(absPrefix) return cls(dirPath=dirPath, name=name, fromVirtualEnv=fromVirtualEnv)
def forModule(cls, name): """ Return an instance of this class representing the module of the given name. If the given module name is "__main__", it will be translated to the actual file name of the top-level script without the .py or .pyc extension. This method assumes that the module with the specified name has already been loaded. """ module = sys.modules[name] filePath = os.path.abspath(module.__file__) filePath = filePath.split(os.path.sep) filePath[-1], extension = os.path.splitext(filePath[-1]) require(extension in ('.py', '.pyc'), 'The name of a user script/module must end in .py or .pyc.') if name == '__main__': log.debug("Discovering real name of module") # User script/module was invoked as the main program if module.__package__: # Invoked as a module via python -m foo.bar log.debug("Script was invoked as a module") name = [filePath.pop()] for package in reversed(module.__package__.split('.')): dirPathTail = filePath.pop() assert dirPathTail == package name.append(dirPathTail) name = '.'.join(reversed(name)) dirPath = os.path.sep.join(filePath) else: # Invoked as a script via python foo/bar.py name = filePath.pop() dirPath = os.path.sep.join(filePath) cls._check_conflict(dirPath, name) else: # User module was imported. Determine the directory containing the top-level package if filePath[-1] == '__init__': # module is a subpackage filePath.pop() for package in reversed(name.split('.')): dirPathTail = filePath.pop() assert dirPathTail == package dirPath = os.path.sep.join(filePath) log.debug("Module dir is %s", dirPath) require(os.path.isdir(dirPath), 'Bad directory path %s for module %s. Note that hot-deployment does not support \ .egg-link files yet, or scripts located in the root directory.', dirPath, name) fromVirtualEnv = inVirtualEnv() and dirPath.startswith(sys.prefix) return cls(dirPath=dirPath, name=name, fromVirtualEnv=fromVirtualEnv)
def forModule(cls, name): """ Return an instance of this class representing the module of the given name. If the given module name is "__main__", it will be translated to the actual file name of the top-level script without the .py or .pyc extension. This method assumes that the module with the specified name has already been loaded. """ module = sys.modules[name] filePath = os.path.abspath(module.__file__) filePath = filePath.split(os.path.sep) filePath[-1], extension = os.path.splitext(filePath[-1]) require(extension in ('.py', '.pyc'), 'The name of a user script/module must end in .py or .pyc.') log.debug("Module name is %s", name) if name == '__main__': log.debug("Discovering real name of module") # User script/module was invoked as the main program if module.__package__: # Invoked as a module via python -m foo.bar log.debug("Script was invoked as a module") name = [filePath.pop()] for package in reversed(module.__package__.split('.')): dirPathTail = filePath.pop() assert dirPathTail == package name.append(dirPathTail) name = '.'.join(reversed(name)) dirPath = os.path.sep.join(filePath) else: # Invoked as a script via python foo/bar.py name = filePath.pop() dirPath = os.path.sep.join(filePath) cls._check_conflict(dirPath, name) else: # User module was imported. Determine the directory containing the top-level package for package in reversed(name.split('.')): dirPathTail = filePath.pop() assert dirPathTail == package dirPath = os.path.sep.join(filePath) log.debug("Module dir is %s", dirPath) require(os.path.isdir(dirPath), 'Bad directory path %s for module %s. Note that hot-deployment does not support \ .egg-link files yet, or scripts located in the root directory.', dirPath, name) fromVirtualEnv = inVirtualEnv() and dirPath.startswith(sys.prefix) return cls(dirPath=dirPath, name=name, fromVirtualEnv=fromVirtualEnv)
def _testExternal(self, moduleName, pyFiles, virtualenv=False): dirPath = self._createTempDir() if virtualenv: self.assertTrue(inVirtualEnv()) # --never-download prevents silent upgrades to pip, wheel and setuptools subprocess.check_call([ 'virtualenv', '--never-download', '--python', exactPython, dirPath ]) sitePackages = os.path.join(dirPath, 'lib', exactPython, 'site-packages') # tuple assignment is necessary to make this line immediately precede the try: oldPrefix, sys.prefix, dirPath = sys.prefix, dirPath, sitePackages else: oldPrefix = None try: for relPath in pyFiles: path = os.path.join(dirPath, relPath) mkdir_p(os.path.dirname(path)) with open(path, 'w') as f: f.write('pass\n') sys.path.append(dirPath) try: userScript = importlib.import_module(moduleName) try: self._test(userScript.__name__, expectedContents=pyFiles, allowExtraContents=True) finally: del userScript while moduleName: del sys.modules[moduleName] self.assertFalse(moduleName in sys.modules) moduleName = '.'.join(moduleName.split('.')[:-1]) finally: sys.path.remove(dirPath) finally: if oldPrefix: sys.prefix = oldPrefix
def _testExternal(self, moduleName, pyFiles, virtualenv=False): dirPath = self._createTempDir() if virtualenv: self.assertTrue(inVirtualEnv()) # --never-download prevents silent upgrades to pip, wheel and setuptools check_call(['virtualenv', '--never-download', dirPath]) sitePackages = os.path.join(dirPath, 'lib', 'python2.7', 'site-packages') # tuple assignment is necessary to make this line immediately precede the try: oldPrefix, sys.prefix, dirPath = sys.prefix, dirPath, sitePackages else: oldPrefix = None try: pycFiles = set(pyFile + 'c' for pyFile in pyFiles) for relPath in pyFiles: path = os.path.join(dirPath, relPath) mkdir_p(os.path.dirname(path)) with open(path, 'w') as f: f.write('pass\n') sys.path.append(dirPath) try: userScript = importlib.import_module(moduleName) try: self._test(userScript.__name__, expectedContents=pycFiles, allowExtraContents=virtualenv) finally: del userScript while moduleName: del sys.modules[moduleName] self.assertFalse(moduleName in sys.modules) moduleName = '.'.join(moduleName.split('.')[:-1]) finally: sys.path.remove(dirPath) finally: if oldPrefix: sys.prefix = oldPrefix