def __init__(self, *args, **kw):
super(BundleCommand, self).__init__(*args, **kw)
# bundle uses different default source and build dirs
build_opt = self.parser.get_option("--build")
build_opt.default = backup_dir(build_prefix, "-bundle")
src_opt = self.parser.get_option("--src")
src_opt.default = backup_dir(src_prefix, "-bundle")
self.parser.set_defaults(**{src_opt.dest: src_opt.default, build_opt.dest: build_opt.default})
def __init__(self, *args, **kw):
super(BundleCommand, self).__init__(*args, **kw)
# bundle uses different default source and build dirs
build_opt = self.parser.get_option("--build")
build_opt.default = backup_dir(build_prefix, '-bundle')
src_opt = self.parser.get_option("--src")
src_opt.default = backup_dir(src_prefix, '-bundle')
self.parser.set_defaults(**{
src_opt.dest: src_opt.default,
build_opt.dest: build_opt.default,
})
def run(self, options, args):
if not args:
raise InstallationError('You must give a bundle filename')
if not options.build_dir:
options.build_dir = backup_dir(build_prefix, '-bundle')
if not options.src_dir:
options.src_dir = backup_dir(src_prefix, '-bundle')
# We have to get everything when creating a bundle:
options.ignore_installed = True
logger.notify('Putting temporary build files in {0!s} and source/develop files in {1!s}'.format(display_path(options.build_dir), display_path(options.src_dir)))
self.bundle_filename = args.pop(0)
requirement_set = super(BundleCommand, self).run(options, args)
return requirement_set
def run(self, options, args):
if not args:
raise InstallationError('You must give a bundle filename')
if not options.build_dir:
options.build_dir = backup_dir(build_prefix, '-bundle')
if not options.src_dir:
options.src_dir = backup_dir(src_prefix, '-bundle')
# We have to get everything when creating a bundle:
options.ignore_installed = True
logger.notify(
'Putting temporary build files in %s and source/develop files in %s'
% (display_path(options.build_dir), display_path(options.src_dir)))
self.bundle_filename = args.pop(0)
requirement_set = super(BundleCommand, self).run(options, args)
return requirement_set
def check_destination(self, dest, url, rev_options, rev_display):
"""
Prepare a location to receive a checkout/clone.
Return True if the location is ready for (and requires) a
checkout/clone, False otherwise.
"""
checkout = True
prompt = False
if os.path.exists(dest):
checkout = False
if os.path.exists(os.path.join(dest, self.dirname)):
existing_url = self.get_url(dest)
if self.compare_urls(existing_url, url):
logger.info('%s in %s exists, and has correct URL (%s)' %
(self.repo_name.title(), display_path(dest),
url))
logger.notify('Updating %s %s%s' %
(display_path(dest), self.repo_name,
rev_display))
self.update(dest, rev_options)
else:
logger.warn('%s %s in %s exists with URL %s' %
(self.name, self.repo_name,
display_path(dest), existing_url))
prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ',
('s', 'i', 'w', 'b'))
else:
logger.warn('Directory %s already exists, '
'and is not a %s %s.' %
(dest, self.name, self.repo_name))
prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b'))
if prompt:
logger.warn('The plan is to install the %s repository %s' %
(self.name, url))
response = ask_path_exists('What to do? %s' % prompt[0],
prompt[1])
if response == 's':
logger.notify('Switching %s %s to %s%s' %
(self.repo_name, display_path(dest), url,
rev_display))
self.switch(dest, url, rev_options)
elif response == 'i':
# do nothing
pass
elif response == 'w':
logger.warn('Deleting %s' % display_path(dest))
rmtree(dest)
checkout = True
elif response == 'b':
dest_dir = backup_dir(dest)
logger.warn('Backing up %s to %s'
% (display_path(dest), dest_dir))
shutil.move(dest, dest_dir)
checkout = True
return checkout
def check_destination(self, dest, url, rev_options, rev_display):
"""
Prepare a location to receive a checkout/clone.
Return True if the location is ready for (and requires) a
checkout/clone, False otherwise.
"""
checkout = True
prompt = False
if os.path.exists(dest):
checkout = False
if os.path.exists(os.path.join(dest, self.dirname)):
existing_url = self.get_url(dest)
if self.compare_urls(existing_url, url):
logger.info(
'%s in %s exists, and has correct URL (%s)' %
(self.repo_name.title(), display_path(dest), url))
logger.notify(
'Updating %s %s%s' %
(display_path(dest), self.repo_name, rev_display))
self.update(dest, rev_options)
else:
logger.warn('%s %s in %s exists with URL %s' %
(self.name, self.repo_name, display_path(dest),
existing_url))
prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ',
('s', 'i', 'w', 'b'))
else:
logger.warn('Directory %s already exists, '
'and is not a %s %s.' %
(dest, self.name, self.repo_name))
prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b'))
if prompt:
logger.warn('The plan is to install the %s repository %s' %
(self.name, url))
response = ask_path_exists('What to do? %s' % prompt[0],
prompt[1])
if response == 's':
logger.notify(
'Switching %s %s to %s%s' %
(self.repo_name, display_path(dest), url, rev_display))
self.switch(dest, url, rev_options)
elif response == 'i':
# do nothing
pass
elif response == 'w':
logger.warn('Deleting %s' % display_path(dest))
rmtree(dest)
checkout = True
elif response == 'b':
dest_dir = backup_dir(dest)
logger.warn('Backing up %s to %s' %
(display_path(dest), dest_dir))
shutil.move(dest, dest_dir)
checkout = True
return checkout
def run(self, options, args):
if not args:
raise InstallationError('You must give a bundle filename')
if not options.build_dir:
options.build_dir = backup_dir(build_prefix, '-bundle')
if not options.src_dir:
options.src_dir = backup_dir(src_prefix, '-bundle')
# We have to get everything when creating a bundle:
options.ignore_installed = True
logger.notify('Putting temporary build files in %s and source/develop files in %s'
% (display_path(options.build_dir), display_path(options.src_dir)))
bundle_filename = args[0]
args = args[1:]
requirement_set = super(BundleCommand, self).run(options, args)
# FIXME: here it has to do something
requirement_set.create_bundle(bundle_filename)
logger.notify('Created bundle in %s' % bundle_filename)
return requirement_set
def zip_package(self, module_name, filename, no_pyc):
orig_filename = filename
logger.notify('Zip %s (in %s)' % (module_name, display_path(filename)))
logger.indent += 2
if filename.endswith('.egg'):
dest_filename = filename
else:
dest_filename = filename + '.zip'
try:
# FIXME: I think this needs to be undoable:
if filename == dest_filename:
filename = backup_dir(orig_filename)
logger.notify(
'Moving %s aside to %s' % (orig_filename, filename)
)
if not self.simulate:
shutil.move(orig_filename, filename)
try:
logger.info(
'Creating zip file in %s' % display_path(dest_filename)
)
if not self.simulate:
zip = zipfile.ZipFile(dest_filename, 'w')
zip.writestr(module_name + '/', '')
for dirpath, dirnames, filenames in os.walk(filename):
if no_pyc:
filenames = [f for f in filenames
if not f.lower().endswith('.pyc')]
for fns, is_dir in [
(dirnames, True), (filenames, False)]:
for fn in fns:
full = os.path.join(dirpath, fn)
dest = os.path.join(
module_name,
dirpath[len(filename):].lstrip(
os.path.sep
),
fn,
)
if is_dir:
zip.writestr(dest + '/', '')
else:
zip.write(full, dest)
zip.close()
logger.info(
'Removing old directory %s' % display_path(filename)
)
if not self.simulate:
rmtree(filename)
except:
# FIXME: need to do an undo here
raise
# FIXME: should also be undone:
self.add_filename_to_pth(dest_filename)
finally:
logger.indent -= 2
def zip_package(self, module_name, filename, no_pyc):
orig_filename = filename
logger.notify('Zip %s (in %s)' % (module_name, display_path(filename)))
logger.indent += 2
if filename.endswith('.egg'):
dest_filename = filename
else:
dest_filename = filename + '.zip'
try:
# FIXME: I think this needs to be undoable:
if filename == dest_filename:
filename = backup_dir(orig_filename)
logger.notify('Moving %s aside to %s' %
(orig_filename, filename))
if not self.simulate:
shutil.move(orig_filename, filename)
try:
logger.info('Creating zip file in %s' %
display_path(dest_filename))
if not self.simulate:
zip = zipfile.ZipFile(dest_filename, 'w')
zip.writestr(module_name + '/', '')
for dirpath, dirnames, filenames in os.walk(filename):
if no_pyc:
filenames = [
f for f in filenames
if not f.lower().endswith('.pyc')
]
for fns, is_dir in [(dirnames, True),
(filenames, False)]:
for fn in fns:
full = os.path.join(dirpath, fn)
dest = os.path.join(
module_name,
dirpath[len(filename):].lstrip(
os.path.sep),
fn,
)
if is_dir:
zip.writestr(dest + '/', '')
else:
zip.write(full, dest)
zip.close()
logger.info('Removing old directory %s' %
display_path(filename))
if not self.simulate:
rmtree(filename)
except:
# FIXME: need to do an undo here
raise
# FIXME: should also be undone:
self.add_filename_to_pth(dest_filename)
finally:
logger.indent -= 2
def _copy_file(filename, location, content_type, link):
copy = True
download_location = os.path.join(location, link.filename)
if os.path.exists(download_location):
response = ask('The file {0!s} exists. (i)gnore, (w)ipe, (b)ackup '.format(display_path(download_location)), ('i', 'w', 'b'))
if response == 'i':
copy = False
elif response == 'w':
logger.warn('Deleting {0!s}'.format(display_path(download_location)))
os.remove(download_location)
elif response == 'b':
dest_file = backup_dir(download_location)
logger.warn('Backing up {0!s} to {1!s}'.format(display_path(download_location), display_path(dest_file)))
shutil.move(download_location, dest_file)
if copy:
shutil.copy(filename, download_location)
logger.indent -= 2
logger.notify('Saved {0!s}'.format(display_path(download_location)))
def _copy_file(filename, location, content_type, link):
copy = True
download_location = os.path.join(location, link.filename)
if os.path.exists(download_location):
response = ask_path_exists(
"The file %s exists. (i)gnore, (w)ipe, (b)ackup " % display_path(download_location), ("i", "w", "b")
)
if response == "i":
copy = False
elif response == "w":
logger.warn("Deleting %s" % display_path(download_location))
os.remove(download_location)
elif response == "b":
dest_file = backup_dir(download_location)
logger.warn("Backing up %s to %s" % (display_path(download_location), display_path(dest_file)))
shutil.move(download_location, dest_file)
if copy:
shutil.copy(filename, download_location)
logger.notify("Saved %s" % display_path(download_location))
def archive(self, build_dir):
assert self.source_dir
create_archive = True
archive_name = '%s-%s.zip' % (self.name, self.installed_version)
archive_path = os.path.join(build_dir, archive_name)
if os.path.exists(archive_path):
response = ask_path_exists(
'The file %s exists. (i)gnore, (w)ipe, (b)ackup ' %
display_path(archive_path), ('i', 'w', 'b'))
if response == 'i':
create_archive = False
elif response == 'w':
logger.warn('Deleting %s' % display_path(archive_path))
os.remove(archive_path)
elif response == 'b':
dest_file = backup_dir(archive_path)
logger.warn(
'Backing up %s to %s' %
(display_path(archive_path), display_path(dest_file))
)
shutil.move(archive_path, dest_file)
if create_archive:
zip = zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED)
dir = os.path.normcase(os.path.abspath(self.source_dir))
for dirpath, dirnames, filenames in os.walk(dir):
if 'pip-egg-info' in dirnames:
dirnames.remove('pip-egg-info')
for dirname in dirnames:
dirname = os.path.join(dirpath, dirname)
name = self._clean_zip_name(dirname, dir)
zipdir = zipfile.ZipInfo(self.name + '/' + name + '/')
zipdir.external_attr = 0x1ED << 16 # 0o755
zip.writestr(zipdir, '')
for filename in filenames:
if filename == PIP_DELETE_MARKER_FILENAME:
continue
filename = os.path.join(dirpath, filename)
name = self._clean_zip_name(filename, dir)
zip.write(filename, self.name + '/' + name)
zip.close()
logger.indent -= 2
logger.notify('Saved %s' % display_path(archive_path))
def archive(self, build_dir):
assert self.source_dir
create_archive = True
archive_name = '%s-%s.zip' % (self.name, self.installed_version)
archive_path = os.path.join(build_dir, archive_name)
if os.path.exists(archive_path):
response = ask_path_exists(
'The file %s exists. (i)gnore, (w)ipe, (b)ackup ' %
display_path(archive_path), ('i', 'w', 'b'))
if response == 'i':
create_archive = False
elif response == 'w':
logger.warn('Deleting %s' % display_path(archive_path))
os.remove(archive_path)
elif response == 'b':
dest_file = backup_dir(archive_path)
logger.warn(
'Backing up %s to %s' %
(display_path(archive_path), display_path(dest_file))
)
shutil.move(archive_path, dest_file)
if create_archive:
zip = zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED)
dir = os.path.normcase(os.path.abspath(self.source_dir))
for dirpath, dirnames, filenames in os.walk(dir):
if 'pip-egg-info' in dirnames:
dirnames.remove('pip-egg-info')
for dirname in dirnames:
dirname = os.path.join(dirpath, dirname)
name = self._clean_zip_name(dirname, dir)
zipdir = zipfile.ZipInfo(self.name + '/' + name + '/')
zipdir.external_attr = 0x1ED << 16 # 0o755
zip.writestr(zipdir, '')
for filename in filenames:
if filename == PIP_DELETE_MARKER_FILENAME:
continue
filename = os.path.join(dirpath, filename)
name = self._clean_zip_name(filename, dir)
zip.write(filename, self.name + '/' + name)
zip.close()
logger.indent -= 2
logger.notify('Saved %s' % display_path(archive_path))
def _copy_file(filename, location, content_type, link):
copy = True
download_location = os.path.join(location, link.filename)
if os.path.exists(download_location):
response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup '
% display_path(download_location), ('i', 'w', 'b'))
if response == 'i':
copy = False
elif response == 'w':
logger.warn('Deleting %s' % display_path(download_location))
os.remove(download_location)
elif response == 'b':
dest_file = backup_dir(download_location)
logger.warn('Backing up %s to %s'
% (display_path(download_location), display_path(dest_file)))
shutil.move(download_location, dest_file)
if copy:
shutil.copy(filename, download_location)
logger.indent -= 2
logger.notify('Saved %s' % display_path(download_location))
def bundle_package(self, cache=True):
"""Makes the pybundle archive (that :program:`pip` can take to
install) with completely resolved dependencies. It yields triple
of package name, filename of the pybundle archive, and its full
path. ::
with build.bundle_package() as (package, filename, path):
sftp.put(path, filename)
:param cache: whether to cache the package file or not.
``True`` by default
:type cache: :class:`bool`
"""
asuka_logger = self.get_logger('bundle_package')
# Makes pip.log.logger to forward records to the standard logging
if not getattr(type(self), 'initialized', False):
type(self).initialized = True
logger.consumers.extend([(Logger.FATAL, asuka_logger.critical),
(Logger.ERROR, asuka_logger.error),
(Logger.WARN, asuka_logger.warn),
(Logger.NOTIFY, asuka_logger.info),
(Logger.INFO, asuka_logger.info),
(Logger.DEBUG, asuka_logger.debug),
(Logger.VERBOSE_DEBUG, asuka_logger.debug)
])
vcs.register(Git)
main_parser = create_main_parser()
bundle = commands['bundle'](main_parser)
with self.archive_package() as (package_name, filename, filepath):
bundle_filename = package_name + '.pybundle'
if cache:
cache_dir_path = os.path.join(tempfile.gettempdir(),
'asuka-pybundle-cache')
if not os.path.isdir(cache_dir_path):
os.makedirs(cache_dir_path)
cache_path = os.path.join(cache_dir_path, bundle_filename)
if os.path.isfile(cache_path):
asuka_logger.info('cache exists: %s, skipping pybundle...',
cache_path)
yield package_name, bundle_filename, cache_path
return
tempdir = tempfile.gettempdir()
bundle_path = os.path.join(os.path.dirname(filepath),
bundle_filename)
asuka_logger.info('pybundle_path = %r', bundle_path)
options = optparse.Values()
options.editables = []
options.requirements = []
options.find_links = []
options.index_url = PYPI_INDEX_URLS[0]
options.extra_index_urls = PYPI_INDEX_URLS[1:]
options.no_index = False
options.use_mirrors = False
options.mirrors = True
options.build_dir = os.path.join(tempdir,
'asuka-dist-build-bundle')
options.target_dir = None
options.download_dir = None
options.download_cache = os.path.join(tempdir,
'asuka-dist-download-cache')
options.src_dir = backup_dir(src_prefix, '-bundle')
options.upgrade = False
options.force_reinstall = False
options.ignore_dependencies = False
options.no_install = True
options.no_download = False
options.install_options = []
options.global_options = []
options.use_user_site = False
options.as_egg = False
asuka_logger.debug('start: pip bundle %s %s', bundle_path,
filepath)
retrial = 0
while 1:
try:
shutil.rmtree(options.build_dir)
except (OSError, IOError):
pass
try:
bundle.run(options, [bundle_path, filepath])
except PipError as e:
asuka_logger.exception(e)
retrial += 1
if retrial < 3:
asuka_logger.error(
'retry pip bundle after %d second(s)... (%d)',
retrial, retrial**2)
options.index_url = PYPI_INDEX_URLS[retrial]
options.extra_index_urls = PYPI_INDEX_URLS[retrial +
1:]
time.sleep(retrial**2)
continue
raise
finally:
if os.path.isdir(options.build_dir):
shutil.rmtree(options.build_dir)
break
asuka_logger.debug('end: pip bundle %s %s', bundle_path, filepath)
if cache:
asuka_logger.info('save pybundle cache %s...', cache_path)
shutil.copyfile(bundle_path, cache_path)
yield package_name, os.path.basename(bundle_path), bundle_path
def bundle_package(self, cache=True):
"""Makes the pybundle archive (that :program:`pip` can take to
install) with completely resolved dependencies. It yields triple
of package name, filename of the pybundle archive, and its full
path. ::
with build.bundle_package() as (package, filename, path):
sftp.put(path, filename)
:param cache: whether to cache the package file or not.
``True`` by default
:type cache: :class:`bool`
"""
asuka_logger = self.get_logger('bundle_package')
# Makes pip.log.logger to forward records to the standard logging
if not getattr(type(self), 'initialized', False):
type(self).initialized = True
logger.consumers.extend([
(Logger.FATAL, asuka_logger.critical),
(Logger.ERROR, asuka_logger.error),
(Logger.WARN, asuka_logger.warn),
(Logger.NOTIFY, asuka_logger.info),
(Logger.INFO, asuka_logger.info),
(Logger.DEBUG, asuka_logger.debug),
(Logger.VERBOSE_DEBUG, asuka_logger.debug)
])
vcs.register(Git)
load_command('bundle')
bundle = command_dict['bundle']
with self.archive_package() as (package_name, filename, filepath):
if cache:
cache_dir_path = os.path.join(
tempfile.gettempdir(),
'asuka-pybundle-cache'
)
if not os.path.isdir(cache_dir_path):
os.makedirs(cache_dir_path)
cache_path = os.path.join(cache_dir_path, filename)
if os.path.isfile(cache_path):
asuka_logger.info('cache exists: %s, skipping pybundle...',
cache_path)
yield package_name, filename, cache_path
return
tempdir = tempfile.gettempdir()
bundle_path = os.path.join(
os.path.dirname(filepath),
package_name + '.pybundle'
)
asuka_logger.info('pybundle_path = %r', bundle_path)
options = optparse.Values()
options.editables = []
options.requirements = []
options.find_links = []
options.index_url = PYPI_INDEX_URLS[0]
options.extra_index_urls = PYPI_INDEX_URLS[1:]
options.no_index = False
options.use_mirrors = False
options.mirrors = True
options.build_dir = os.path.join(
tempdir,
'asuka-dist-build-bundle'
)
options.target_dir = None
options.download_dir = None
options.download_cache = os.path.join(
tempdir,
'asuka-dist-download-cache'
)
options.src_dir = backup_dir(src_prefix, '-bundle')
options.upgrade = False
options.force_reinstall = False
options.ignore_dependencies = False
options.no_install = True
options.no_download = False
options.install_options = []
options.global_options = []
options.use_user_site = False
options.as_egg = False
asuka_logger.debug('start: pip bundle %s %s', bundle_path, filepath)
retrial = 0
while 1:
try:
shutil.rmtree(options.build_dir)
except (OSError, IOError):
pass
try:
bundle.run(options, [bundle_path, filepath])
except PipError as e:
asuka_logger.exception(e)
retrial += 1
if retrial < 3:
asuka_logger.error(
'retry pip bundle after %d second(s)... (%d)',
retrial, retrial ** 2
)
options.index_url = PYPI_INDEX_URLS[retrial]
options.extra_index_urls = PYPI_INDEX_URLS[retrial+1:]
time.sleep(retrial ** 2)
continue
raise
finally:
shutil.rmtree(options.build_dir)
break
asuka_logger.debug('end: pip bundle %s %s', bundle_path, filepath)
if cache:
asuka_logger.info('save pybundle cache %s...', cache_path)
shutil.copyfile(filepath, cache_path)
yield package_name, os.path.basename(bundle_path), bundle_path
