def save_to_egg(entry_pts, version=None, py_dir=None, src_dir=None,
src_files=None, dst_dir=None, logger=None, observer=None,
need_requirements=True):
"""
Save state and other files to an egg. Analyzes the objects saved for
distribution dependencies. Modules not found in any distribution are
recorded in an ``egg-info/openmdao_orphans.txt`` file. Also creates and
saves loader scripts for each entry point.
entry_pts: list
List of ``(obj, obj_name, obj_group)`` tuples.
The first of these specifies the root object and package name.
version: string
Defaults to a timestamp of the form 'YYYY.MM.DD.HH.mm'.
py_dir: string
The (root) directory for local Python files.
It defaults to the current directory.
src_dir: string
The root of all (relative) `src_files`.
dst_dir: string
The directory to write the egg in.
observer: callable
Will be called via an :class:`EggObserver` intermediary.
need_requirements: bool
If True, distributions required by the egg will be determined.
This can be set False if the egg is just being used for distribution
within the local host.
Returns ``(egg_filename, required_distributions, orphan_modules)``.
"""
root, name, group = entry_pts[0]
logger = logger or NullLogger()
observer = eggobserver.EggObserver(observer, logger)
orig_dir = os.getcwd()
if not py_dir:
py_dir = orig_dir
else:
py_dir = os.path.realpath(py_dir)
if sys.platform == 'win32': #pragma no cover
py_dir = py_dir.lower()
if src_dir:
src_dir = os.path.realpath(src_dir)
if sys.platform == 'win32': #pragma no cover
src_dir = src_dir.lower()
src_files = src_files or set()
if not version:
now = datetime.datetime.now() # Could consider using utcnow().
version = '%d.%02d.%02d.%02d.%02d' % \
(now.year, now.month, now.day, now.hour, now.minute)
dst_dir = dst_dir or orig_dir
if not os.access(dst_dir, os.W_OK):
msg = "Can't save to '%s', no write permission" % dst_dir
observer.exception(msg)
raise IOError(msg)
egg_name = eggwriter.egg_filename(name, version)
logger.debug('Saving to %s in %s...', egg_name, orig_dir)
# Get a list of all objects we'll be saving.
objs = _get_objects(root, logger)
# Check that each object can be pickled.
_check_objects(objs, logger, observer)
# Fixup objects, classes, & sys.modules for __main__ imports.
fixup = _fix_objects(objs, observer)
# Verify that the fixups are retained.
_verify_objects(root, logger, observer)
tmp_dir = None
try:
if need_requirements:
# Determine distributions and local modules required.
required_distributions, local_modules, orphan_modules = \
_get_distributions(objs, py_dir, logger, observer)
else:
required_distributions = set()
local_modules = set()
orphan_modules = set()
# Collect Python modules.
for dirpath, dirnames, filenames in os.walk(py_dir):
dirs = copy.copy(dirnames)
for path in dirs:
if not os.path.exists(os.path.join(dirpath, path,
'__init__.py')):
dirnames.remove(path)
for path in filenames:
if path.endswith('.py'):
local_modules.add(os.path.join(dirpath, path))
# Ensure module corresponding to __main__ is local if it was used.
# (Test script embedded in egg is an example of how this might occur)
fixup_objects, fixup_classes, fixup_modules = fixup
if fixup_objects: #pragma no cover
# Something references __main__.
main_mod = sys.modules['__main__'].__file__
local_modules.add(os.path.abspath(main_mod))
logger.debug(' py_dir: %s', py_dir)
logger.debug(' src_dir: %s', src_dir)
logger.debug(' local_modules:')
for module in sorted(local_modules):
mod = module
if mod.startswith(py_dir):
mod = mod[len(py_dir)+1:]
logger.debug(' %s', mod)
# Move to scratch area.
tmp_dir = tempfile.mkdtemp(prefix='Egg_', dir=tmp_dir)
os.chdir(tmp_dir)
os.mkdir(name)
cleanup_files = []
try:
# Copy external files from src_dir.
if src_dir:
for path in src_files:
subdir = os.path.dirname(path)
if subdir:
subdir = os.path.join(name, subdir)
if not os.path.exists(subdir):
os.makedirs(subdir)
src = os.path.join(src_dir, path)
dst = os.path.join(name, path)
if sys.platform == 'win32': #pragma no cover
shutil.copy2(src, dst)
else:
os.symlink(src, dst)
# Copy local modules from py_dir.
for path in local_modules:
if not os.path.exists(
os.path.join(name, os.path.basename(path))):
if not os.path.isabs(path):
path = os.path.join(py_dir, path)
shutil.copy(path, name)
# For each entry point...
entry_info = []
for obj, obj_name, obj_group in entry_pts:
clean_name = obj_name
if clean_name.startswith(name+'.'):
clean_name = clean_name[len(name)+1:]
clean_name = clean_name.replace('.', '_')
# Save state of object hierarchy.
state_name, state_path = \
_write_state_file(name, obj, clean_name, logger, observer)
src_files.add(state_name)
cleanup_files.append(state_path)
# Create loader script.
loader = '%s_loader' % clean_name
loader_path = os.path.join(name, loader+'.py')
cleanup_files.append(loader_path)
_write_loader_script(loader_path, state_name, name, obj is root)
entry_info.append((obj_group, obj_name, loader))
# If needed, make an empty __init__.py
init_path = os.path.join(name, '__init__.py')
if not os.path.exists(init_path):
cleanup_files.append(init_path)
out = open(init_path, 'w')
out.close()
# Save everything to an egg.
doc = root.__doc__ or ''
entry_map = _create_entry_map(entry_info)
orphans = [mod for mod, path in orphan_modules]
eggwriter.write(name, version, doc, entry_map,
src_files, required_distributions,
orphans, dst_dir, logger, observer.observer)
finally:
for path in cleanup_files:
if os.path.exists(path):
os.remove(path)
finally:
os.chdir(orig_dir)
if tmp_dir:
shutil.rmtree(tmp_dir)
_restore_objects(fixup)
return (egg_name, required_distributions, orphan_modules)
def save_to_egg(entry_pts,
version=None,
py_dir=None,
src_dir=None,
src_files=None,
dst_dir=None,
logger=None,
observer=None,
need_requirements=True):
"""
Save state and other files to an egg. Analyzes the objects saved for
distribution dependencies. Modules not found in any distribution are
recorded in an ``egg-info/openmdao_orphans.txt`` file. Also creates and
saves loader scripts for each entry point.
entry_pts: list
List of ``(obj, obj_name, obj_group)`` tuples.
The first of these specifies the root object and package name.
version: string
Defaults to a timestamp of the form 'YYYY.MM.DD.HH.mm'.
py_dir: string
The (root) directory for local Python files.
It defaults to the current directory.
src_dir: string
The root of all (relative) `src_files`.
dst_dir: string
The directory to write the egg in.
observer: callable
Will be called via an :class:`EggObserver` intermediary.
need_requirements: bool
If True, distributions required by the egg will be determined.
This can be set False if the egg is just being used for distribution
within the local host.
Returns ``(egg_filename, required_distributions, orphan_modules)``.
"""
root, name, group = entry_pts[0]
logger = logger or NullLogger()
observer = eggobserver.EggObserver(observer, logger)
orig_dir = os.getcwd()
if not py_dir:
py_dir = orig_dir
else:
py_dir = os.path.realpath(py_dir)
if sys.platform == 'win32': # pragma no cover
py_dir = py_dir.lower()
if src_dir:
src_dir = os.path.realpath(src_dir)
if sys.platform == 'win32': # pragma no cover
src_dir = src_dir.lower()
src_files = src_files or set()
if not version:
now = datetime.datetime.now() # Could consider using utcnow().
version = '%d.%02d.%02d.%02d.%02d' % \
(now.year, now.month, now.day, now.hour, now.minute)
dst_dir = dst_dir or orig_dir
if not os.access(dst_dir, os.W_OK):
msg = "Can't save to '%s', no write permission" % dst_dir
observer.exception(msg)
raise IOError(msg)
egg_name = eggwriter.egg_filename(name, version)
logger.debug('Saving to %s in %s...', egg_name, orig_dir)
# Clone __main__ as a 'real' module and have classes reference that.
_fix_main(logger)
# Get a list of all objects we'll be saving.
objs = _get_objects(root, logger)
# Check that each object can be pickled.
_check_objects(objs, logger, observer)
# Verify that no __main__ references are still around.
_verify_objects(root, logger, observer)
tmp_dir = None
try:
if need_requirements:
# Determine distributions and local modules required.
required_distributions, local_modules, orphan_modules = \
_get_distributions(objs, py_dir, logger, observer)
else:
required_distributions = set()
local_modules = set()
orphan_modules = set()
# Collect Python modules.
for dirpath, dirnames, filenames in os.walk(py_dir):
dirs = copy.copy(dirnames)
for path in dirs:
if not os.path.exists(
os.path.join(dirpath, path, '__init__.py')):
dirnames.remove(path)
for path in filenames:
if path.endswith('.py'):
local_modules.add(os.path.join(dirpath, path))
# Ensure module defining root is local. Saving a root which is defined
# in a package can be hidden if the package was installed.
root_mod = root.__class__.__module__
root_mod = sys.modules[root_mod].__file__
if root_mod.endswith('.pyc') or root_mod.endswith('.pyo'):
root_mod = root_mod[:-1]
local_modules.add(os.path.abspath(root_mod))
logger.log(LOG_DEBUG2, ' py_dir: %s', py_dir)
logger.log(LOG_DEBUG2, ' src_dir: %s', src_dir)
logger.log(LOG_DEBUG2, ' local_modules:')
for module in sorted(local_modules):
mod = module
if mod.startswith(py_dir):
mod = mod[len(py_dir) + 1:]
logger.log(LOG_DEBUG2, ' %s', mod)
# Move to scratch area.
tmp_dir = tempfile.mkdtemp(prefix='Egg_', dir=tmp_dir)
os.chdir(tmp_dir)
os.mkdir(name)
cleanup_files = []
try:
# Copy external files from src_dir.
if src_dir:
for path in src_files:
subdir = os.path.dirname(path)
if subdir:
subdir = os.path.join(name, subdir)
if not os.path.exists(subdir):
os.makedirs(subdir)
src = os.path.join(src_dir, path)
dst = os.path.join(name, path)
if sys.platform == 'win32': # pragma no cover
shutil.copy2(src, dst)
else:
os.symlink(src, dst)
# Copy local modules from py_dir.
for path in local_modules:
if not os.path.exists(
os.path.join(name, os.path.basename(path))):
if not os.path.isabs(path):
path = os.path.join(py_dir, path)
shutil.copy(path, name)
# For each entry point...
entry_info = []
for obj, obj_name, obj_group in entry_pts:
clean_name = obj_name
if clean_name.startswith(name + '.'):
clean_name = clean_name[len(name) + 1:]
clean_name = clean_name.replace('.', '_')
# Save state of object hierarchy.
state_name, state_path = \
_write_state_file(name, obj, clean_name, logger, observer)
src_files.add(state_name)
cleanup_files.append(state_path)
# Create loader script.
loader = '%s_loader' % clean_name
loader_path = os.path.join(name, loader + '.py')
cleanup_files.append(loader_path)
_write_loader_script(loader_path, state_name, name,
obj is root)
entry_info.append((obj_group, obj_name, loader))
# If needed, make an empty __init__.py
init_path = os.path.join(name, '__init__.py')
if not os.path.exists(init_path):
cleanup_files.append(init_path)
out = open(init_path, 'w')
out.close()
# Save everything to an egg.
doc = root.__doc__ or ''
entry_map = _create_entry_map(entry_info)
orphans = [mod for mod, path in orphan_modules]
eggwriter.write(name, version, doc, entry_map, src_files,
required_distributions, orphans, dst_dir, logger,
observer.observer)
finally:
for path in cleanup_files:
if os.path.exists(path):
os.remove(path)
finally:
os.chdir(orig_dir)
if tmp_dir:
shutil.rmtree(tmp_dir, onerror=onerror)
return (egg_name, required_distributions, orphan_modules)