def __init__(self, *args, **kw): super(BundleCommand, self).__init__(*args, **kw) # bundle uses different default source and build dirs build_opt = self.parser.get_option("--build") build_opt.default = backup_dir(build_prefix, "-bundle") src_opt = self.parser.get_option("--src") src_opt.default = backup_dir(src_prefix, "-bundle") self.parser.set_defaults(**{src_opt.dest: src_opt.default, build_opt.dest: build_opt.default})
def __init__(self, *args, **kw): super(BundleCommand, self).__init__(*args, **kw) # bundle uses different default source and build dirs build_opt = self.parser.get_option("--build") build_opt.default = backup_dir(build_prefix, '-bundle') src_opt = self.parser.get_option("--src") src_opt.default = backup_dir(src_prefix, '-bundle') self.parser.set_defaults(**{ src_opt.dest: src_opt.default, build_opt.dest: build_opt.default, })
def run(self, options, args): if not args: raise InstallationError('You must give a bundle filename') if not options.build_dir: options.build_dir = backup_dir(build_prefix, '-bundle') if not options.src_dir: options.src_dir = backup_dir(src_prefix, '-bundle') # We have to get everything when creating a bundle: options.ignore_installed = True logger.notify('Putting temporary build files in {0!s} and source/develop files in {1!s}'.format(display_path(options.build_dir), display_path(options.src_dir))) self.bundle_filename = args.pop(0) requirement_set = super(BundleCommand, self).run(options, args) return requirement_set
def run(self, options, args): if not args: raise InstallationError('You must give a bundle filename') if not options.build_dir: options.build_dir = backup_dir(build_prefix, '-bundle') if not options.src_dir: options.src_dir = backup_dir(src_prefix, '-bundle') # We have to get everything when creating a bundle: options.ignore_installed = True logger.notify( 'Putting temporary build files in %s and source/develop files in %s' % (display_path(options.build_dir), display_path(options.src_dir))) self.bundle_filename = args.pop(0) requirement_set = super(BundleCommand, self).run(options, args) return requirement_set
def check_destination(self, dest, url, rev_options, rev_display): """ Prepare a location to receive a checkout/clone. Return True if the location is ready for (and requires) a checkout/clone, False otherwise. """ checkout = True prompt = False if os.path.exists(dest): checkout = False if os.path.exists(os.path.join(dest, self.dirname)): existing_url = self.get_url(dest) if self.compare_urls(existing_url, url): logger.info('%s in %s exists, and has correct URL (%s)' % (self.repo_name.title(), display_path(dest), url)) logger.notify('Updating %s %s%s' % (display_path(dest), self.repo_name, rev_display)) self.update(dest, rev_options) else: logger.warn('%s %s in %s exists with URL %s' % (self.name, self.repo_name, display_path(dest), existing_url)) prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b')) else: logger.warn('Directory %s already exists, ' 'and is not a %s %s.' % (dest, self.name, self.repo_name)) prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b')) if prompt: logger.warn('The plan is to install the %s repository %s' % (self.name, url)) response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) if response == 's': logger.notify('Switching %s %s to %s%s' % (self.repo_name, display_path(dest), url, rev_display)) self.switch(dest, url, rev_options) elif response == 'i': # do nothing pass elif response == 'w': logger.warn('Deleting %s' % display_path(dest)) rmtree(dest) checkout = True elif response == 'b': dest_dir = backup_dir(dest) logger.warn('Backing up %s to %s' % (display_path(dest), dest_dir)) shutil.move(dest, dest_dir) checkout = True return checkout
def check_destination(self, dest, url, rev_options, rev_display): """ Prepare a location to receive a checkout/clone. Return True if the location is ready for (and requires) a checkout/clone, False otherwise. """ checkout = True prompt = False if os.path.exists(dest): checkout = False if os.path.exists(os.path.join(dest, self.dirname)): existing_url = self.get_url(dest) if self.compare_urls(existing_url, url): logger.info( '%s in %s exists, and has correct URL (%s)' % (self.repo_name.title(), display_path(dest), url)) logger.notify( 'Updating %s %s%s' % (display_path(dest), self.repo_name, rev_display)) self.update(dest, rev_options) else: logger.warn('%s %s in %s exists with URL %s' % (self.name, self.repo_name, display_path(dest), existing_url)) prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b')) else: logger.warn('Directory %s already exists, ' 'and is not a %s %s.' % (dest, self.name, self.repo_name)) prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b')) if prompt: logger.warn('The plan is to install the %s repository %s' % (self.name, url)) response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) if response == 's': logger.notify( 'Switching %s %s to %s%s' % (self.repo_name, display_path(dest), url, rev_display)) self.switch(dest, url, rev_options) elif response == 'i': # do nothing pass elif response == 'w': logger.warn('Deleting %s' % display_path(dest)) rmtree(dest) checkout = True elif response == 'b': dest_dir = backup_dir(dest) logger.warn('Backing up %s to %s' % (display_path(dest), dest_dir)) shutil.move(dest, dest_dir) checkout = True return checkout
def run(self, options, args): if not args: raise InstallationError('You must give a bundle filename') if not options.build_dir: options.build_dir = backup_dir(build_prefix, '-bundle') if not options.src_dir: options.src_dir = backup_dir(src_prefix, '-bundle') # We have to get everything when creating a bundle: options.ignore_installed = True logger.notify('Putting temporary build files in %s and source/develop files in %s' % (display_path(options.build_dir), display_path(options.src_dir))) bundle_filename = args[0] args = args[1:] requirement_set = super(BundleCommand, self).run(options, args) # FIXME: here it has to do something requirement_set.create_bundle(bundle_filename) logger.notify('Created bundle in %s' % bundle_filename) return requirement_set
def zip_package(self, module_name, filename, no_pyc): orig_filename = filename logger.notify('Zip %s (in %s)' % (module_name, display_path(filename))) logger.indent += 2 if filename.endswith('.egg'): dest_filename = filename else: dest_filename = filename + '.zip' try: # FIXME: I think this needs to be undoable: if filename == dest_filename: filename = backup_dir(orig_filename) logger.notify( 'Moving %s aside to %s' % (orig_filename, filename) ) if not self.simulate: shutil.move(orig_filename, filename) try: logger.info( 'Creating zip file in %s' % display_path(dest_filename) ) if not self.simulate: zip = zipfile.ZipFile(dest_filename, 'w') zip.writestr(module_name + '/', '') for dirpath, dirnames, filenames in os.walk(filename): if no_pyc: filenames = [f for f in filenames if not f.lower().endswith('.pyc')] for fns, is_dir in [ (dirnames, True), (filenames, False)]: for fn in fns: full = os.path.join(dirpath, fn) dest = os.path.join( module_name, dirpath[len(filename):].lstrip( os.path.sep ), fn, ) if is_dir: zip.writestr(dest + '/', '') else: zip.write(full, dest) zip.close() logger.info( 'Removing old directory %s' % display_path(filename) ) if not self.simulate: rmtree(filename) except: # FIXME: need to do an undo here raise # FIXME: should also be undone: self.add_filename_to_pth(dest_filename) finally: logger.indent -= 2
def zip_package(self, module_name, filename, no_pyc): orig_filename = filename logger.notify('Zip %s (in %s)' % (module_name, display_path(filename))) logger.indent += 2 if filename.endswith('.egg'): dest_filename = filename else: dest_filename = filename + '.zip' try: # FIXME: I think this needs to be undoable: if filename == dest_filename: filename = backup_dir(orig_filename) logger.notify('Moving %s aside to %s' % (orig_filename, filename)) if not self.simulate: shutil.move(orig_filename, filename) try: logger.info('Creating zip file in %s' % display_path(dest_filename)) if not self.simulate: zip = zipfile.ZipFile(dest_filename, 'w') zip.writestr(module_name + '/', '') for dirpath, dirnames, filenames in os.walk(filename): if no_pyc: filenames = [ f for f in filenames if not f.lower().endswith('.pyc') ] for fns, is_dir in [(dirnames, True), (filenames, False)]: for fn in fns: full = os.path.join(dirpath, fn) dest = os.path.join( module_name, dirpath[len(filename):].lstrip( os.path.sep), fn, ) if is_dir: zip.writestr(dest + '/', '') else: zip.write(full, dest) zip.close() logger.info('Removing old directory %s' % display_path(filename)) if not self.simulate: rmtree(filename) except: # FIXME: need to do an undo here raise # FIXME: should also be undone: self.add_filename_to_pth(dest_filename) finally: logger.indent -= 2
def _copy_file(filename, location, content_type, link): copy = True download_location = os.path.join(location, link.filename) if os.path.exists(download_location): response = ask('The file {0!s} exists. (i)gnore, (w)ipe, (b)ackup '.format(display_path(download_location)), ('i', 'w', 'b')) if response == 'i': copy = False elif response == 'w': logger.warn('Deleting {0!s}'.format(display_path(download_location))) os.remove(download_location) elif response == 'b': dest_file = backup_dir(download_location) logger.warn('Backing up {0!s} to {1!s}'.format(display_path(download_location), display_path(dest_file))) shutil.move(download_location, dest_file) if copy: shutil.copy(filename, download_location) logger.indent -= 2 logger.notify('Saved {0!s}'.format(display_path(download_location)))
def _copy_file(filename, location, content_type, link): copy = True download_location = os.path.join(location, link.filename) if os.path.exists(download_location): response = ask_path_exists( "The file %s exists. (i)gnore, (w)ipe, (b)ackup " % display_path(download_location), ("i", "w", "b") ) if response == "i": copy = False elif response == "w": logger.warn("Deleting %s" % display_path(download_location)) os.remove(download_location) elif response == "b": dest_file = backup_dir(download_location) logger.warn("Backing up %s to %s" % (display_path(download_location), display_path(dest_file))) shutil.move(download_location, dest_file) if copy: shutil.copy(filename, download_location) logger.notify("Saved %s" % display_path(download_location))
def archive(self, build_dir): assert self.source_dir create_archive = True archive_name = '%s-%s.zip' % (self.name, self.installed_version) archive_path = os.path.join(build_dir, archive_name) if os.path.exists(archive_path): response = ask_path_exists( 'The file %s exists. (i)gnore, (w)ipe, (b)ackup ' % display_path(archive_path), ('i', 'w', 'b')) if response == 'i': create_archive = False elif response == 'w': logger.warn('Deleting %s' % display_path(archive_path)) os.remove(archive_path) elif response == 'b': dest_file = backup_dir(archive_path) logger.warn( 'Backing up %s to %s' % (display_path(archive_path), display_path(dest_file)) ) shutil.move(archive_path, dest_file) if create_archive: zip = zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED) dir = os.path.normcase(os.path.abspath(self.source_dir)) for dirpath, dirnames, filenames in os.walk(dir): if 'pip-egg-info' in dirnames: dirnames.remove('pip-egg-info') for dirname in dirnames: dirname = os.path.join(dirpath, dirname) name = self._clean_zip_name(dirname, dir) zipdir = zipfile.ZipInfo(self.name + '/' + name + '/') zipdir.external_attr = 0x1ED << 16 # 0o755 zip.writestr(zipdir, '') for filename in filenames: if filename == PIP_DELETE_MARKER_FILENAME: continue filename = os.path.join(dirpath, filename) name = self._clean_zip_name(filename, dir) zip.write(filename, self.name + '/' + name) zip.close() logger.indent -= 2 logger.notify('Saved %s' % display_path(archive_path))
def _copy_file(filename, location, content_type, link): copy = True download_location = os.path.join(location, link.filename) if os.path.exists(download_location): response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup ' % display_path(download_location), ('i', 'w', 'b')) if response == 'i': copy = False elif response == 'w': logger.warn('Deleting %s' % display_path(download_location)) os.remove(download_location) elif response == 'b': dest_file = backup_dir(download_location) logger.warn('Backing up %s to %s' % (display_path(download_location), display_path(dest_file))) shutil.move(download_location, dest_file) if copy: shutil.copy(filename, download_location) logger.indent -= 2 logger.notify('Saved %s' % display_path(download_location))
def bundle_package(self, cache=True): """Makes the pybundle archive (that :program:`pip` can take to install) with completely resolved dependencies. It yields triple of package name, filename of the pybundle archive, and its full path. :: with build.bundle_package() as (package, filename, path): sftp.put(path, filename) :param cache: whether to cache the package file or not. ``True`` by default :type cache: :class:`bool` """ asuka_logger = self.get_logger('bundle_package') # Makes pip.log.logger to forward records to the standard logging if not getattr(type(self), 'initialized', False): type(self).initialized = True logger.consumers.extend([(Logger.FATAL, asuka_logger.critical), (Logger.ERROR, asuka_logger.error), (Logger.WARN, asuka_logger.warn), (Logger.NOTIFY, asuka_logger.info), (Logger.INFO, asuka_logger.info), (Logger.DEBUG, asuka_logger.debug), (Logger.VERBOSE_DEBUG, asuka_logger.debug) ]) vcs.register(Git) main_parser = create_main_parser() bundle = commands['bundle'](main_parser) with self.archive_package() as (package_name, filename, filepath): bundle_filename = package_name + '.pybundle' if cache: cache_dir_path = os.path.join(tempfile.gettempdir(), 'asuka-pybundle-cache') if not os.path.isdir(cache_dir_path): os.makedirs(cache_dir_path) cache_path = os.path.join(cache_dir_path, bundle_filename) if os.path.isfile(cache_path): asuka_logger.info('cache exists: %s, skipping pybundle...', cache_path) yield package_name, bundle_filename, cache_path return tempdir = tempfile.gettempdir() bundle_path = os.path.join(os.path.dirname(filepath), bundle_filename) asuka_logger.info('pybundle_path = %r', bundle_path) options = optparse.Values() options.editables = [] options.requirements = [] options.find_links = [] options.index_url = PYPI_INDEX_URLS[0] options.extra_index_urls = PYPI_INDEX_URLS[1:] options.no_index = False options.use_mirrors = False options.mirrors = True options.build_dir = os.path.join(tempdir, 'asuka-dist-build-bundle') options.target_dir = None options.download_dir = None options.download_cache = os.path.join(tempdir, 'asuka-dist-download-cache') options.src_dir = backup_dir(src_prefix, '-bundle') options.upgrade = False options.force_reinstall = False options.ignore_dependencies = False options.no_install = True options.no_download = False options.install_options = [] options.global_options = [] options.use_user_site = False options.as_egg = False asuka_logger.debug('start: pip bundle %s %s', bundle_path, filepath) retrial = 0 while 1: try: shutil.rmtree(options.build_dir) except (OSError, IOError): pass try: bundle.run(options, [bundle_path, filepath]) except PipError as e: asuka_logger.exception(e) retrial += 1 if retrial < 3: asuka_logger.error( 'retry pip bundle after %d second(s)... (%d)', retrial, retrial**2) options.index_url = PYPI_INDEX_URLS[retrial] options.extra_index_urls = PYPI_INDEX_URLS[retrial + 1:] time.sleep(retrial**2) continue raise finally: if os.path.isdir(options.build_dir): shutil.rmtree(options.build_dir) break asuka_logger.debug('end: pip bundle %s %s', bundle_path, filepath) if cache: asuka_logger.info('save pybundle cache %s...', cache_path) shutil.copyfile(bundle_path, cache_path) yield package_name, os.path.basename(bundle_path), bundle_path
def bundle_package(self, cache=True): """Makes the pybundle archive (that :program:`pip` can take to install) with completely resolved dependencies. It yields triple of package name, filename of the pybundle archive, and its full path. :: with build.bundle_package() as (package, filename, path): sftp.put(path, filename) :param cache: whether to cache the package file or not. ``True`` by default :type cache: :class:`bool` """ asuka_logger = self.get_logger('bundle_package') # Makes pip.log.logger to forward records to the standard logging if not getattr(type(self), 'initialized', False): type(self).initialized = True logger.consumers.extend([ (Logger.FATAL, asuka_logger.critical), (Logger.ERROR, asuka_logger.error), (Logger.WARN, asuka_logger.warn), (Logger.NOTIFY, asuka_logger.info), (Logger.INFO, asuka_logger.info), (Logger.DEBUG, asuka_logger.debug), (Logger.VERBOSE_DEBUG, asuka_logger.debug) ]) vcs.register(Git) load_command('bundle') bundle = command_dict['bundle'] with self.archive_package() as (package_name, filename, filepath): if cache: cache_dir_path = os.path.join( tempfile.gettempdir(), 'asuka-pybundle-cache' ) if not os.path.isdir(cache_dir_path): os.makedirs(cache_dir_path) cache_path = os.path.join(cache_dir_path, filename) if os.path.isfile(cache_path): asuka_logger.info('cache exists: %s, skipping pybundle...', cache_path) yield package_name, filename, cache_path return tempdir = tempfile.gettempdir() bundle_path = os.path.join( os.path.dirname(filepath), package_name + '.pybundle' ) asuka_logger.info('pybundle_path = %r', bundle_path) options = optparse.Values() options.editables = [] options.requirements = [] options.find_links = [] options.index_url = PYPI_INDEX_URLS[0] options.extra_index_urls = PYPI_INDEX_URLS[1:] options.no_index = False options.use_mirrors = False options.mirrors = True options.build_dir = os.path.join( tempdir, 'asuka-dist-build-bundle' ) options.target_dir = None options.download_dir = None options.download_cache = os.path.join( tempdir, 'asuka-dist-download-cache' ) options.src_dir = backup_dir(src_prefix, '-bundle') options.upgrade = False options.force_reinstall = False options.ignore_dependencies = False options.no_install = True options.no_download = False options.install_options = [] options.global_options = [] options.use_user_site = False options.as_egg = False asuka_logger.debug('start: pip bundle %s %s', bundle_path, filepath) retrial = 0 while 1: try: shutil.rmtree(options.build_dir) except (OSError, IOError): pass try: bundle.run(options, [bundle_path, filepath]) except PipError as e: asuka_logger.exception(e) retrial += 1 if retrial < 3: asuka_logger.error( 'retry pip bundle after %d second(s)... (%d)', retrial, retrial ** 2 ) options.index_url = PYPI_INDEX_URLS[retrial] options.extra_index_urls = PYPI_INDEX_URLS[retrial+1:] time.sleep(retrial ** 2) continue raise finally: shutil.rmtree(options.build_dir) break asuka_logger.debug('end: pip bundle %s %s', bundle_path, filepath) if cache: asuka_logger.info('save pybundle cache %s...', cache_path) shutil.copyfile(filepath, cache_path) yield package_name, os.path.basename(bundle_path), bundle_path