def write(name,
version,
doc,
entry_map,
src_files,
distributions,
modules,
dst_dir,
logger,
observer=None,
compress=True):
"""
Write egg in the manner of :mod:`setuptools`, with some differences:
- Writes directly to the zip file, avoiding some intermediate copies.
- Doesn't compile any Python modules.
name: string
Must be an alphanumeric string.
version: string
Must be an alphanumeric string.
doc: string
Used for the `Summary` and `Description` entries in the egg's metadata.
entry_map: dict
A :mod:`pkg_resources` :class:`EntryPoint` map: a dictionary mapping
group names to dictionaries mapping entry point names to
:class:`EntryPoint` objects.
src_files: list
List of non-Python files to include.
distributions: list
List of Distributions this egg depends on. It is used for the `Requires`
entry in the egg's metadata.
modules: list
List of module names not found in a distribution that this egg depends
on. It is used for the `Requires` entry in the egg's metadata and is
also recorded in the 'openmdao_orphans.txt' resource.
dst_dir: string
The directory to write the egg to.
logger: Logger
Used for recording progress, etc.
observer: callable
Will be called via an :class:`EggObserver` intermediary.
Returns the egg's filename.
"""
observer = eggobserver.EggObserver(observer, logger)
egg_name = egg_filename(name, version)
egg_path = os.path.join(dst_dir, egg_name)
distributions = sorted(distributions, key=lambda dist: dist.project_name)
modules = sorted(modules)
sources = []
files = []
size = 0 # Approximate (uncompressed) size. Used to set allowZip64 flag.
# Collect src_files.
for path in src_files:
path = os.path.join(name, path)
files.append(path)
size += os.path.getsize(path)
# Collect Python modules.
for dirpath, dirnames, filenames in os.walk('.', followlinks=True):
dirs = copy.copy(dirnames)
for path in dirs:
if not os.path.exists(os.path.join(dirpath, path, '__init__.py')):
dirnames.remove(path)
for path in filenames:
if path.endswith('.py'):
path = os.path.join(dirpath[2:], path) # Skip leading './'
files.append(path)
size += os.path.getsize(path)
sources.append(path)
# Package info -> EGG-INFO/PKG-INFO
pkg_info = []
pkg_info.append('Metadata-Version: 1.1')
pkg_info.append('Name: %s' % pkg_resources.safe_name(name))
pkg_info.append('Version: %s' % pkg_resources.safe_version(version))
pkg_info.append('Summary: %s' % doc.strip().split('\n')[0])
pkg_info.append('Description: %s' % doc.strip())
pkg_info.append('Author-email: UNKNOWN')
pkg_info.append('License: UNKNOWN')
pkg_info.append('Platform: UNKNOWN')
for dist in distributions:
pkg_info.append('Requires: %s (%s)' %
(dist.project_name, dist.version))
for module in modules:
pkg_info.append('Requires: %s' % module)
pkg_info = '\n'.join(pkg_info) + '\n'
sources.append(name + '.egg-info/PKG-INFO')
size += len(pkg_info)
# Dependency links -> EGG-INFO/dependency_links.txt
dependency_links = '\n'
sources.append(name + '.egg-info/dependency_links.txt')
size += len(dependency_links)
# Entry points -> EGG-INFO/entry_points.txt
entry_points = []
for entry_group in sorted(entry_map.keys()):
entry_points.append('[%s]' % entry_group)
for entry_name in sorted(entry_map[entry_group].keys()):
entry_points.append('%s' % entry_map[entry_group][entry_name])
entry_points.append('')
entry_points = '\n'.join(entry_points) + '\n'
sources.append(name + '.egg-info/entry_points.txt')
size += len(entry_points)
# Unsafe -> EGG-INFO/not-zip-safe
not_zip_safe = '\n'
sources.append(name + '.egg-info/not-zip-safe')
size += len(not_zip_safe)
# Requirements -> EGG-INFO/requires.txt
requirements = [str(dist.as_requirement()) for dist in distributions]
requirements = '\n'.join(requirements) + '\n'
sources.append(name + '.egg-info/requires.txt')
size += len(requirements)
# Modules not part of a distribution -> EGG-INFO/openmdao_orphans.txt
orphans = '\n'.join(modules) + '\n'
sources.append(name + '.egg-info/openmdao_orphans.txt')
size += len(orphans)
# Top-level names -> EGG-INFO/top_level.txt
top_level = '%s\n' % name
sources.append(name + '.egg-info/top_level.txt')
size += len(top_level)
# Manifest -> EGG-INFO/SOURCES.txt
sources.append(name + '.egg-info/SOURCES.txt')
sources = '\n'.join(sorted(sources)) + '\n'
size += len(sources)
# Open zipfile.
logger.debug('Creating %s', egg_path)
zip64 = size > zipfile.ZIP64_LIMIT
compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED
egg = zipfile.ZipFile(egg_path, 'w', compression, zip64)
stats = {
'completed_files': 0.,
'total_files': float(8 + len(files)),
'completed_bytes': 0.,
'total_bytes': float(size)
}
# Write egg info.
_write_info(egg, 'PKG-INFO', pkg_info, observer, stats)
_write_info(egg, 'dependency_links.txt', dependency_links, observer, stats)
_write_info(egg, 'entry_points.txt', entry_points, observer, stats)
_write_info(egg, 'not-zip-safe', not_zip_safe, observer, stats)
_write_info(egg, 'requires.txt', requirements, observer, stats)
_write_info(egg, 'openmdao_orphans.txt', orphans, observer, stats)
_write_info(egg, 'top_level.txt', top_level, observer, stats)
_write_info(egg, 'SOURCES.txt', sources, observer, stats)
# Write collected files.
for path in sorted(files):
_write_file(egg, path, observer, stats)
observer.complete(egg_name)
egg.close()
if os.path.getsize(egg_path) > zipfile.ZIP64_LIMIT:
logger.warning('Egg zipfile requires Zip64 support to unzip.')
return egg_name
def save_to_egg(entry_pts,
version=None,
py_dir=None,
src_dir=None,
src_files=None,
dst_dir=None,
logger=None,
observer=None,
need_requirements=True):
"""
Save state and other files to an egg. Analyzes the objects saved for
distribution dependencies. Modules not found in any distribution are
recorded in an ``egg-info/openmdao_orphans.txt`` file. Also creates and
saves loader scripts for each entry point.
entry_pts: list
List of ``(obj, obj_name, obj_group)`` tuples.
The first of these specifies the root object and package name.
version: string
Defaults to a timestamp of the form 'YYYY.MM.DD.HH.mm'.
py_dir: string
The (root) directory for local Python files.
It defaults to the current directory.
src_dir: string
The root of all (relative) `src_files`.
dst_dir: string
The directory to write the egg in.
observer: callable
Will be called via an :class:`EggObserver` intermediary.
need_requirements: bool
If True, distributions required by the egg will be determined.
This can be set False if the egg is just being used for distribution
within the local host.
Returns ``(egg_filename, required_distributions, orphan_modules)``.
"""
root, name, group = entry_pts[0]
logger = logger or NullLogger()
observer = eggobserver.EggObserver(observer, logger)
orig_dir = os.getcwd()
if not py_dir:
py_dir = orig_dir
else:
py_dir = os.path.realpath(py_dir)
if sys.platform == 'win32': # pragma no cover
py_dir = py_dir.lower()
if src_dir:
src_dir = os.path.realpath(src_dir)
if sys.platform == 'win32': # pragma no cover
src_dir = src_dir.lower()
src_files = src_files or set()
if not version:
now = datetime.datetime.now() # Could consider using utcnow().
version = '%d.%02d.%02d.%02d.%02d' % \
(now.year, now.month, now.day, now.hour, now.minute)
dst_dir = dst_dir or orig_dir
if not os.access(dst_dir, os.W_OK):
msg = "Can't save to '%s', no write permission" % dst_dir
observer.exception(msg)
raise IOError(msg)
egg_name = eggwriter.egg_filename(name, version)
logger.debug('Saving to %s in %s...', egg_name, orig_dir)
# Clone __main__ as a 'real' module and have classes reference that.
_fix_main(logger)
# Get a list of all objects we'll be saving.
objs = _get_objects(root, logger)
# Check that each object can be pickled.
_check_objects(objs, logger, observer)
# Verify that no __main__ references are still around.
_verify_objects(root, logger, observer)
tmp_dir = None
try:
if need_requirements:
# Determine distributions and local modules required.
required_distributions, local_modules, orphan_modules = \
_get_distributions(objs, py_dir, logger, observer)
else:
required_distributions = set()
local_modules = set()
orphan_modules = set()
# Collect Python modules.
for dirpath, dirnames, filenames in os.walk(py_dir):
dirs = copy.copy(dirnames)
for path in dirs:
if not os.path.exists(
os.path.join(dirpath, path, '__init__.py')):
dirnames.remove(path)
for path in filenames:
if path.endswith('.py'):
local_modules.add(os.path.join(dirpath, path))
# Ensure module defining root is local. Saving a root which is defined
# in a package can be hidden if the package was installed.
root_mod = root.__class__.__module__
root_mod = sys.modules[root_mod].__file__
if root_mod.endswith('.pyc') or root_mod.endswith('.pyo'):
root_mod = root_mod[:-1]
local_modules.add(os.path.abspath(root_mod))
logger.log(LOG_DEBUG2, ' py_dir: %s', py_dir)
logger.log(LOG_DEBUG2, ' src_dir: %s', src_dir)
logger.log(LOG_DEBUG2, ' local_modules:')
for module in sorted(local_modules):
mod = module
if mod.startswith(py_dir):
mod = mod[len(py_dir) + 1:]
logger.log(LOG_DEBUG2, ' %s', mod)
# Move to scratch area.
tmp_dir = tempfile.mkdtemp(prefix='Egg_', dir=tmp_dir)
os.chdir(tmp_dir)
os.mkdir(name)
cleanup_files = []
try:
# Copy external files from src_dir.
if src_dir:
for path in src_files:
subdir = os.path.dirname(path)
if subdir:
subdir = os.path.join(name, subdir)
if not os.path.exists(subdir):
os.makedirs(subdir)
src = os.path.join(src_dir, path)
dst = os.path.join(name, path)
if sys.platform == 'win32': # pragma no cover
shutil.copy2(src, dst)
else:
os.symlink(src, dst)
# Copy local modules from py_dir.
for path in local_modules:
if not os.path.exists(
os.path.join(name, os.path.basename(path))):
if not os.path.isabs(path):
path = os.path.join(py_dir, path)
shutil.copy(path, name)
# For each entry point...
entry_info = []
for obj, obj_name, obj_group in entry_pts:
clean_name = obj_name
if clean_name.startswith(name + '.'):
clean_name = clean_name[len(name) + 1:]
clean_name = clean_name.replace('.', '_')
# Save state of object hierarchy.
state_name, state_path = \
_write_state_file(name, obj, clean_name, logger, observer)
src_files.add(state_name)
cleanup_files.append(state_path)
# Create loader script.
loader = '%s_loader' % clean_name
loader_path = os.path.join(name, loader + '.py')
cleanup_files.append(loader_path)
_write_loader_script(loader_path, state_name, name,
obj is root)
entry_info.append((obj_group, obj_name, loader))
# If needed, make an empty __init__.py
init_path = os.path.join(name, '__init__.py')
if not os.path.exists(init_path):
cleanup_files.append(init_path)
out = open(init_path, 'w')
out.close()
# Save everything to an egg.
doc = root.__doc__ or ''
entry_map = _create_entry_map(entry_info)
orphans = [mod for mod, path in orphan_modules]
eggwriter.write(name, version, doc, entry_map, src_files,
required_distributions, orphans, dst_dir, logger,
observer.observer)
finally:
for path in cleanup_files:
if os.path.exists(path):
os.remove(path)
finally:
os.chdir(orig_dir)
if tmp_dir:
shutil.rmtree(tmp_dir, onerror=onerror)
return (egg_name, required_distributions, orphan_modules)