def create(cls, jobStore, leaderPath):
"""
Saves the content of the file or directory at the given path to the given job store
and returns a resource object representing that content for the purpose of obtaining it
again at a generic, public URL. This method should be invoked on the leader node.
:rtype: Resource
"""
if os.path.isdir(leaderPath):
if inVirtualEnv() and leaderPath.startswith(sys.prefix):
subcls = VirtualEnvResource
else:
subcls = DirectoryResource
elif os.path.isfile(leaderPath):
subcls = FileResource
elif os.path.exists(leaderPath):
raise AssertionError("Neither a file or a directory: '%s'" % leaderPath)
else:
raise AssertionError("No such file or directory: '%s'" % leaderPath)
pathHash = subcls._pathHash(leaderPath)
contentHash = hashlib.md5()
# noinspection PyProtectedMember
with subcls._load(leaderPath) as src:
with jobStore.writeSharedFileStream(pathHash, isProtected=False) as dst:
userScript = src.read()
contentHash.update(userScript)
dst.write(userScript)
return subcls(name=os.path.basename(leaderPath),
pathHash=pathHash,
url=(jobStore.getSharedPublicUrl(pathHash)),
contentHash=contentHash.hexdigest())
def create(cls, jobStore, leaderPath):
"""
Saves the content of the file or directory at the given path to the given job store
and returns a resource object representing that content for the purpose of obtaining it
again at a generic, public URL. This method should be invoked on the leader node.
:rtype: Resource
"""
if os.path.isdir(leaderPath):
if inVirtualEnv() and leaderPath.startswith(sys.prefix):
subcls = VirtualEnvResource
else:
subcls = DirectoryResource
elif os.path.isfile(leaderPath):
subcls = FileResource
elif os.path.exists(leaderPath):
raise AssertionError("Neither a file or a directory: '%s'" %
leaderPath)
else:
raise AssertionError("No such file or directory: '%s'" %
leaderPath)
pathHash = subcls._pathHash(leaderPath)
contentHash = hashlib.md5()
# noinspection PyProtectedMember
with subcls._load(leaderPath) as src:
with jobStore.writeSharedFileStream(pathHash,
isProtected=False) as dst:
userScript = src.read()
contentHash.update(userScript)
dst.write(userScript)
return subcls(name=os.path.basename(leaderPath),
pathHash=pathHash,
url=(jobStore.getSharedPublicUrl(pathHash)),
contentHash=contentHash.hexdigest())
def forModule(cls, name: str) -> Any:
"""
Return an instance of this class representing the module of the given name.
If the given module name is "__main__", it will be translated to the actual
file name of the top-level script without the .py or .pyc extension. This
method assumes that the module with the specified name has already been loaded.
"""
module = sys.modules[name]
filePath = os.path.abspath(module.__file__)
filePath = filePath.split(os.path.sep)
filePath[-1], extension = os.path.splitext(filePath[-1])
if extension not in (".py", ".pyc"):
raise Exception(
"The name of a user script/module must end in .py or .pyc.")
if name == "__main__":
logger.debug("Discovering real name of module")
# User script/module was invoked as the main program
if module.__package__:
# Invoked as a module via python -m foo.bar
logger.debug("Script was invoked as a module")
name = [filePath.pop()]
for package in reversed(module.__package__.split('.')):
dirPathTail = filePath.pop()
assert dirPathTail == package
name.append(dirPathTail)
name = '.'.join(reversed(name))
dirPath = os.path.sep.join(filePath)
else:
# Invoked as a script via python foo/bar.py
name = filePath.pop()
dirPath = os.path.sep.join(filePath)
cls._check_conflict(dirPath, name)
else:
# User module was imported. Determine the directory containing the top-level package
if filePath[-1] == '__init__':
# module is a subpackage
filePath.pop()
for package in reversed(name.split('.')):
dirPathTail = filePath.pop()
assert dirPathTail == package
dirPath = os.path.abspath(os.path.sep.join(filePath))
absPrefix = os.path.abspath(sys.prefix)
inVenv = inVirtualEnv()
logger.debug("Module dir is %s, our prefix is %s, virtualenv: %s",
dirPath, absPrefix, inVenv)
if not os.path.isdir(dirPath):
raise Exception(
f'Bad directory path {dirPath} for module {name}. Note that hot-deployment does not support .egg-link files yet, or scripts located in the root directory.'
)
fromVirtualEnv = inVenv and dirPath.startswith(absPrefix)
return cls(dirPath=dirPath, name=name, fromVirtualEnv=fromVirtualEnv)
def forModule(cls, name):
"""
Return an instance of this class representing the module of the given name. If the given
module name is "__main__", it will be translated to the actual file name of the top-level
script without the .py or .pyc extension. This method assumes that the module with the
specified name has already been loaded.
"""
module = sys.modules[name]
filePath = os.path.abspath(module.__file__)
filePath = filePath.split(os.path.sep)
filePath[-1], extension = os.path.splitext(filePath[-1])
require(extension in ('.py', '.pyc'),
'The name of a user script/module must end in .py or .pyc.')
if name == '__main__':
log.debug("Discovering real name of module")
# User script/module was invoked as the main program
if module.__package__:
# Invoked as a module via python -m foo.bar
log.debug("Script was invoked as a module")
name = [filePath.pop()]
for package in reversed(module.__package__.split('.')):
dirPathTail = filePath.pop()
assert dirPathTail == package
name.append(dirPathTail)
name = '.'.join(reversed(name))
dirPath = os.path.sep.join(filePath)
else:
# Invoked as a script via python foo/bar.py
name = filePath.pop()
dirPath = os.path.sep.join(filePath)
cls._check_conflict(dirPath, name)
else:
# User module was imported. Determine the directory containing the top-level package
if filePath[-1] == '__init__':
# module is a subpackage
filePath.pop()
for package in reversed(name.split('.')):
dirPathTail = filePath.pop()
assert dirPathTail == package
dirPath = os.path.sep.join(filePath)
log.debug("Module dir is %s", dirPath)
require(os.path.isdir(dirPath),
'Bad directory path %s for module %s. Note that hot-deployment does not support \
.egg-link files yet, or scripts located in the root directory.', dirPath, name)
fromVirtualEnv = inVirtualEnv() and dirPath.startswith(sys.prefix)
return cls(dirPath=dirPath, name=name, fromVirtualEnv=fromVirtualEnv)
def forModule(cls, name):
"""
Return an instance of this class representing the module of the given name. If the given
module name is "__main__", it will be translated to the actual file name of the top-level
script without the .py or .pyc extension. This method assumes that the module with the
specified name has already been loaded.
"""
module = sys.modules[name]
filePath = os.path.abspath(module.__file__)
filePath = filePath.split(os.path.sep)
filePath[-1], extension = os.path.splitext(filePath[-1])
require(extension in ('.py', '.pyc'),
'The name of a user script/module must end in .py or .pyc.')
log.debug("Module name is %s", name)
if name == '__main__':
log.debug("Discovering real name of module")
# User script/module was invoked as the main program
if module.__package__:
# Invoked as a module via python -m foo.bar
log.debug("Script was invoked as a module")
name = [filePath.pop()]
for package in reversed(module.__package__.split('.')):
dirPathTail = filePath.pop()
assert dirPathTail == package
name.append(dirPathTail)
name = '.'.join(reversed(name))
dirPath = os.path.sep.join(filePath)
else:
# Invoked as a script via python foo/bar.py
name = filePath.pop()
dirPath = os.path.sep.join(filePath)
cls._check_conflict(dirPath, name)
else:
# User module was imported. Determine the directory containing the top-level package
for package in reversed(name.split('.')):
dirPathTail = filePath.pop()
assert dirPathTail == package
dirPath = os.path.sep.join(filePath)
log.debug("Module dir is %s", dirPath)
require(os.path.isdir(dirPath),
'Bad directory path %s for module %s. Note that hot-deployment does not support \
.egg-link files yet, or scripts located in the root directory.', dirPath, name)
fromVirtualEnv = inVirtualEnv() and dirPath.startswith(sys.prefix)
return cls(dirPath=dirPath, name=name, fromVirtualEnv=fromVirtualEnv)
def _testExternal(self, moduleName, pyFiles, virtualenv=False):
dirPath = self._createTempDir()
if virtualenv:
self.assertTrue(inVirtualEnv())
# --never-download prevents silent upgrades to pip, wheel and setuptools
subprocess.check_call([
'virtualenv', '--never-download', '--python', exactPython,
dirPath
])
sitePackages = os.path.join(dirPath, 'lib', exactPython,
'site-packages')
# tuple assignment is necessary to make this line immediately precede the try:
oldPrefix, sys.prefix, dirPath = sys.prefix, dirPath, sitePackages
else:
oldPrefix = None
try:
for relPath in pyFiles:
path = os.path.join(dirPath, relPath)
mkdir_p(os.path.dirname(path))
with open(path, 'w') as f:
f.write('pass\n')
sys.path.append(dirPath)
try:
userScript = importlib.import_module(moduleName)
try:
self._test(userScript.__name__,
expectedContents=pyFiles,
allowExtraContents=True)
finally:
del userScript
while moduleName:
del sys.modules[moduleName]
self.assertFalse(moduleName in sys.modules)
moduleName = '.'.join(moduleName.split('.')[:-1])
finally:
sys.path.remove(dirPath)
finally:
if oldPrefix:
sys.prefix = oldPrefix
def _testExternal(self, moduleName, pyFiles, virtualenv=False):
dirPath = self._createTempDir()
if virtualenv:
self.assertTrue(inVirtualEnv())
# --never-download prevents silent upgrades to pip, wheel and setuptools
check_call(['virtualenv', '--never-download', dirPath])
sitePackages = os.path.join(dirPath, 'lib', 'python2.7', 'site-packages')
# tuple assignment is necessary to make this line immediately precede the try:
oldPrefix, sys.prefix, dirPath = sys.prefix, dirPath, sitePackages
else:
oldPrefix = None
try:
pycFiles = set(pyFile + 'c' for pyFile in pyFiles)
for relPath in pyFiles:
path = os.path.join(dirPath, relPath)
mkdir_p(os.path.dirname(path))
with open(path, 'w') as f:
f.write('pass\n')
sys.path.append(dirPath)
try:
userScript = importlib.import_module(moduleName)
try:
self._test(userScript.__name__,
expectedContents=pycFiles,
allowExtraContents=virtualenv)
finally:
del userScript
while moduleName:
del sys.modules[moduleName]
self.assertFalse(moduleName in sys.modules)
moduleName = '.'.join(moduleName.split('.')[:-1])
finally:
sys.path.remove(dirPath)
finally:
if oldPrefix:
sys.prefix = oldPrefix
