def update(self, dest, rev_options): # First fetch changes from the default remote call_subprocess([self.cmd, 'fetch', '-q'], cwd=dest) # Then reset to wanted revision (maby even origin/master) if rev_options: rev_options = self.check_rev_options(rev_options[0], dest, rev_options) call_subprocess([self.cmd, 'reset', '--hard', '-q'] + rev_options, cwd=dest)
def obtain(self, dest): url, rev = self.get_url_rev() rev_options = get_rev_options(url, rev) if rev: rev_display = " (to revision %s)" % rev else: rev_display = "" if self.check_destination(dest, url, rev_options, rev_display): logger.notify("Checking out %s%s to %s" % (url, rev_display, display_path(dest))) call_subprocess([self.cmd, "checkout", "-q"] + rev_options + [url, dest])
def export(self, location): """Export the Hg repository at the url to the destination location""" temp_dir = tempfile.mkdtemp('-export', 'pip-') self.unpack(temp_dir) try: call_subprocess( [self.cmd, 'archive', location], filter_stdout=self._filter, show_stdout=False, cwd=temp_dir) finally: rmtree(temp_dir)
def get_tag_revs(self, location): tags = call_subprocess( [self.cmd, 'tag'], show_stdout=False, cwd=location) tag_revs = [] for line in tags.splitlines(): tag = line.strip() rev = call_subprocess( [self.cmd, 'rev-parse', tag], show_stdout=False, cwd=location) tag_revs.append((rev.strip(), tag)) tag_revs = dict(tag_revs) return tag_revs
def obtain(self, dest): url, rev = self.get_url_rev() rev_options = get_rev_options(url, rev) if rev: rev_display = ' (to revision %s)' % rev else: rev_display = '' if self.check_destination(dest, url, rev_options, rev_display): logger.notify('Checking out %s%s to %s' % (url, rev_display, display_path(dest))) call_subprocess( [self.cmd, 'checkout', '-q'] + rev_options + [url, dest])
def obtain(self, dest): url, rev = self.get_url_rev() if rev: rev_options = [rev] rev_display = ' (to revision {0!s})'.format(rev) else: rev_options = [] rev_display = '' if self.check_destination(dest, url, rev_options, rev_display): logger.notify('Cloning hg {0!s}{1!s} to {2!s}'.format(url, rev_display, display_path(dest))) call_subprocess([self.cmd, 'clone', '--noupdate', '-q', url, dest]) call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest)
def get_branch_revs(self, location): branches = call_subprocess( [self.cmd, 'branch', '-r'], show_stdout=False, cwd=location) branch_revs = [] for line in branches.splitlines(): line = line.split('->')[0].strip() branch = "".join([b for b in line.split() if b != '*']) rev = call_subprocess( [self.cmd, 'rev-parse', branch], show_stdout=False, cwd=location) branch_revs.append((rev.strip(), branch)) branch_revs = dict(branch_revs) return branch_revs
def obtain(self, dest): url, rev = self.get_url_rev() if rev: rev_options = ['-r', rev] rev_display = ' (to revision {0!s})'.format(rev) else: rev_options = [] rev_display = '' if self.check_destination(dest, url, rev_options, rev_display): logger.notify('Checking out {0!s}{1!s} to {2!s}'.format(url, rev_display, display_path(dest))) call_subprocess( [self.cmd, 'checkout', '-q'] + rev_options + [url, dest])
def export(self, location): """Export the Bazaar repository at the url to the destination location""" temp_dir = tempfile.mkdtemp('-export', 'pip-') self.unpack(temp_dir) if os.path.exists(location): # Remove the location to make sure Bazaar can export it correctly rmtree(location) try: call_subprocess([self.cmd, 'export', location], cwd=temp_dir, filter_stdout=self._filter, show_stdout=False) finally: rmtree(temp_dir)
def export(self, location): """Export the Git repository at the url to the destination location""" temp_dir = tempfile.mkdtemp('-export', 'pip-') self.unpack(temp_dir) try: if not location.endswith('/'): location = location + '/' call_subprocess( [self.cmd, 'checkout-index', '-a', '-f', '--prefix', location], filter_stdout=self._filter, show_stdout=False, cwd=temp_dir) finally: rmtree(temp_dir)
def obtain(self, dest): url, rev = self.get_url_rev() if rev: rev_options = [rev] rev_display = ' (to revision %s)' % rev else: rev_options = [] rev_display = '' if self.check_destination(dest, url, rev_options, rev_display): logger.notify('Cloning hg %s%s to %s' % (url, rev_display, display_path(dest))) call_subprocess([self.cmd, 'clone', '--noupdate', '-q', url, dest]) call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest)
def upload_to_repository(self, options): current = os.path.abspath(os.curdir) opts = ['sdist', 'register', '-r', options.req_repository, 'upload', '-r', options.req_repository] setup = ' '.join(['setup.py'] + opts) logger.notify('Running %s for packages in %s' % (setup, options.req_cache_dir)) pkgs = self._individual_packages(options) logger.indent += 2 for n in pkgs: os.chdir(n) call_subprocess([sys.executable, 'setup.py'] + opts) os.chdir(current) logger.indent -= 2
def unpack(self, location): """Get the bzr branch at the url to the destination location""" url, rev = self.get_url_rev() logger.notify('Checking out bzr repository %s to %s' % (url, location)) logger.indent += 2 try: if os.path.exists(location): os.rmdir(location) call_subprocess( [self.cmd, 'branch', url, location], filter_stdout=self._filter, show_stdout=False) finally: logger.indent -= 2
def unpack(self, location): """Clone the Hg repository at the url to the destination location""" url, rev = self.get_url_rev() logger.notify('Cloning Mercurial repository %s to %s' % (url, location)) logger.indent += 2 try: if os.path.exists(location): os.rmdir(location) call_subprocess( [self.cmd, 'clone', url, location], filter_stdout=self._filter, show_stdout=False) finally: logger.indent -= 2
def export(self, location): """Export the svn repository at the url to the destination location""" url, rev = self.get_url_rev() logger.notify('Exporting svn repository %s to %s' % (url, location)) logger.indent += 2 try: if os.path.exists(location): # Subversion doesn't like to check out over an existing directory # --force fixes this, but was only added in svn 1.5 rmtree(location) call_subprocess( [self.cmd, 'export', url, location], filter_stdout=self._filter, show_stdout=False) finally: logger.indent -= 2
def obtain(self, dest): url, rev = self.get_url_rev() if rev: rev_options = [rev] rev_display = ' (to %s)' % rev else: rev_options = ['origin/master'] rev_display = '' if self.check_destination(dest, url, rev_options, rev_display): logger.notify('Cloning %s%s to %s' % (url, rev_display, display_path(dest))) call_subprocess( [self.cmd, 'clone', '-q', url, dest]) rev_options = self.check_rev_options(rev, dest, rev_options) call_subprocess( [self.cmd, 'checkout', '-q'] + rev_options, cwd=dest)
def switch(self, dest, url, rev_options): repo_config = os.path.join(dest, self.dirname, 'hgrc') config = ConfigParser.SafeConfigParser() try: config.read(repo_config) config.set('paths', 'default', url) config_file = open(repo_config, 'w') config.write(config_file) config_file.close() except (OSError, ConfigParser.NoSectionError): e = sys.exc_info()[1] logger.warn( 'Could not switch Mercurial repository to {0!s}: {1!s}'.format(url, e)) else: call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest)
def get_url(self, location): url = call_subprocess( [self.cmd, 'showconfig', 'paths.default'], show_stdout=False, cwd=location).strip() if self._is_local_repository(url): url = path_to_url2(url) return url.strip()
def get_url(self, location): url = call_subprocess( [self.cmd, 'showconfig', 'paths.default'], show_stdout=False, cwd=location).strip() if url.startswith('/') or url.startswith('\\'): url = path_to_url(url) return url.strip()
def _get_svn_url_rev(self, location): f = open(os.path.join(location, self.dirname, "entries")) data = f.read() f.close() if data.startswith("8") or data.startswith("9") or data.startswith("10"): data = list(map(str.splitlines, data.split("\n\x0c\n"))) del data[0][0] # get rid of the '8' url = data[0][3] revs = [int(d[9]) for d in data if len(d) > 9 and d[9]] + [0] elif data.startswith("<?xml"): match = _svn_xml_url_re.search(data) if not match: raise ValueError("Badly formatted data: %r" % data) url = match.group(1) # get repository URL revs = [int(m.group(1)) for m in _svn_rev_re.finditer(data)] + [0] else: try: # subversion >= 1.7 xml = call_subprocess([self.cmd, "info", "--xml", location], show_stdout=False) url = _svn_info_xml_url_re.search(xml).group(1) revs = [int(m.group(1)) for m in _svn_info_xml_rev_re.finditer(xml)] except InstallationError: url, revs = None, [] if revs: rev = max(revs) else: rev = 0 return url, rev
def obtain(self, dest): url, rev = self.get_url_rev() if rev: rev_options = [rev] rev_display = ' (to {0!s})'.format(rev) else: rev_options = ['origin/master'] rev_display = '' if self.check_destination(dest, url, rev_options, rev_display): logger.notify('Cloning {0!s}{1!s} to {2!s}'.format(url, rev_display, display_path(dest))) call_subprocess([self.cmd, 'clone', '-q', url, dest]) if rev: rev_options = self.check_rev_options(rev, dest, rev_options) # Only do a checkout if rev_options differs from HEAD if not self.get_revision(dest).startswith(rev_options[0]): call_subprocess([self.cmd, 'checkout', '-q'] + rev_options, cwd=dest)
def unpack(self, location): """Clone the Git repository at a specific revision""" url, rev = self.get_url_rev() logger.notify('Cloning Git repository %s @%s to %s' % (url, rev, location)) logger.indent += 2 try: if os.path.exists(location): os.rmdir(location) call_subprocess( [self.cmd, 'clone', '-n', url, location], filter_stdout=self._filter, show_stdout=False) call_subprocess( [self.cmd, 'checkout', rev], filter_stdout=self._filter, show_stdout=False, cwd=location) finally: logger.indent -= 2
def get_tag_revs(self, svn_tag_url): stdout = call_subprocess([self.cmd, "ls", "-v", svn_tag_url], show_stdout=False) results = [] for line in stdout.splitlines(): parts = line.split() rev = int(parts[0]) tag = parts[-1].strip("/") results.append((tag, rev)) return results
def get_url(self, location): urls = call_subprocess( [self.cmd, 'info'], show_stdout=False, cwd=location) for line in urls.splitlines(): line = line.strip() for x in ('checkout of branch: ', 'parent branch: '): if line.startswith(x): return line.split(x)[1] return None
def get_tag_revs(self, svn_tag_url): stdout = call_subprocess( ['svn', 'ls', '-v', svn_tag_url], show_stdout=False) results = [] for line in stdout.splitlines(): parts = line.split() rev = int(parts[0]) tag = parts[-1].strip('/') results.append((tag, rev)) return results
def install_reqs(self, require_virtualenv=True): # @@@ move to using Python pip APIs and not relying on the OS if sys.platform == "win32": PIP_CMD = "pip.exe" else: PIP_CMD = "pip" pip_cmd = resolve_command(PIP_CMD) requirements_file = os.path.join(self.project_dir, "requirements.txt") environ = {} if require_virtualenv: environ["PIP_REQUIRE_VIRTUALENV"] = "true" pip.call_subprocess([ pip_cmd, "install", "--requirement", requirements_file, ], show_stdout=True, extra_environ=environ)
def get_tag_revs(self, location): tags = call_subprocess( [self.cmd, 'tags'], show_stdout=False, cwd=location) tag_revs = [] for line in tags.splitlines(): tags_match = re.search(r'([.\w-]+)\s*(.*)$', line) if tags_match: tag = tags_match.group(1) rev = tags_match.group(2) tag_revs.append((rev.strip(), tag.strip())) return dict(tag_revs)
def build_release(dist_dir, kind, user, repository, commit): basename = build_basename(user, repository, commit) if kind == "github": source_dir = os.path.join(WORK_DIR, basename) elif kind == "bitbucket": source_dir = os.path.join(WORK_DIR, repository) setup_py = os.path.realpath(os.path.join(source_dir, "setup.py")) # use setuptools hack to allow egg_info in setup.cfg to work for # development builds (requires setuptools in dev release environment) cmd = [ sys.executable, "-c", "import setuptools;__file__=%r;execfile(%r)" % (setup_py, setup_py), "sdist", "-d", dist_dir, ] call_subprocess(cmd, cwd = source_dir, show_stdout = False, command_desc = "python setup.py sdist", )
def get_branch_revs(self, location): branches = call_subprocess( [self.cmd, 'branches'], show_stdout=False, cwd=location) branch_revs = [] for line in branches.splitlines(): branches_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line) if branches_match: branch = branches_match.group(1) rev = branches_match.group(2) branch_revs.append((rev.strip(), branch.strip())) return dict(branch_revs)
def get_tag_revs(self, location): tags = call_subprocess([self.cmd, 'tags'], show_stdout=False, cwd=location) tag_revs = [] for line in tags.splitlines(): tags_match = re.search(r'([.\w-]+)\s*(.*)$', line) if tags_match: tag = tags_match.group(1) rev = tags_match.group(2) tag_revs.append((rev.strip(), tag.strip())) return dict(tag_revs)
def get_branch_revs(self, location): branches = call_subprocess([self.cmd, 'branches'], show_stdout=False, cwd=location) branch_revs = [] for line in branches.splitlines(): branches_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line) if branches_match: branch = branches_match.group(1) rev = branches_match.group(2) if "default" != branch: branch_revs.append((rev.strip(), branch.strip())) return dict(branch_revs)
def get_url(self, location): urls = call_subprocess( [self.cmd, 'info'], show_stdout=False, cwd=location) for line in urls.splitlines(): line = line.strip() for x in ('checkout of branch: ', 'parent branch: '): if line.startswith(x): repo = line.split(x)[1] if self._is_local_repository(repo): return path_to_url2(repo) return repo return None
def get_url(self, location): urls = call_subprocess([self.cmd, 'info'], show_stdout=False, cwd=location) for line in urls.splitlines(): line = line.strip() for x in ('checkout of branch: ', 'parent branch: '): if line.startswith(x): repo = line.split(x)[1] if self._is_local_repository(repo): return path_to_url2(repo) return repo return None
def install_reqs(self, require_virtualenv=True): # @@@ move to using Python pip APIs and not relying on the OS if sys.platform == "win32": PIP_CMD = "pip.exe" else: PIP_CMD = "pip" pip_cmd = resolve_command(PIP_CMD) requirements_file = os.path.join(self.project_dir, "requirements", "project.txt") environ = {} if require_virtualenv: environ["PIP_REQUIRE_VIRTUALENV"] = "true" pip.call_subprocess([ pip_cmd, "install", "--requirement", requirements_file, ], show_stdout=True, extra_environ=environ)
def get_remote_refs(self, *ls_remote_args): """ Return an iterator over triples (pipversion, ref, SHA1) for each symbolic reference in the repository, where pipversion is a version number in pip format, ref is the symbolic name for that revision in the repository, and SHA1 is that revision's immutable revision number. """ url, rev = self.get_url_rev() remote_refs = call_subprocess( (self.cmd, 'ls-remote') + ls_remote_args + (url, ), show_stdout=False) return ((ref.rsplit('/', 1)[-1], ref, sha) for (sha, ref) in (line.strip().split() for line in remote_refs.splitlines()))
def get_info(self, location): """Returns (url, revision), where both are strings""" assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location output = call_subprocess( [self.cmd, 'info', location], show_stdout=False, extra_environ={'LANG': 'C'}) match = _svn_url_re.search(output) if not match: logger.warn('Cannot determine URL of svn checkout %s' % display_path(location)) logger.info('Output that cannot be parsed: \n%s' % output) return None, None url = match.group(1).strip() match = _svn_revision_re.search(output) if not match: logger.warn('Cannot determine revision of svn checkout %s' % display_path(location)) logger.info('Output that cannot be parsed: \n%s' % output) return url, None return url, match.group(1)
def run_command( cls, cmd, # type: List[str] show_stdout=True, # type: bool cwd=None, # type: Optional[str] on_returncode='raise', # type: str extra_ok_returncodes=None, # type: Optional[Iterable[int]] command_desc=None, # type: Optional[str] extra_environ=None, # type: Optional[Mapping[str, Any]] spinner=None # type: Optional[SpinnerInterface] ): # type: (...) -> Optional[Text] """ Run a VCS subcommand This is simply a wrapper around call_subprocess that adds the VCS command name, and checks that the VCS is available """ cmd = [cls.name] + cmd try: return call_subprocess(cmd, show_stdout, cwd, on_returncode=on_returncode, extra_ok_returncodes=extra_ok_returncodes, command_desc=command_desc, extra_environ=extra_environ, unset_environ=cls.unset_environ, spinner=spinner) except OSError as e: # errno.ENOENT = no such file or directory # In other words, the VCS executable isn't available if e.errno == errno.ENOENT: raise BadCommand( 'Cannot find command %r - do you have ' '%r installed and in your ' 'PATH?' % (cls.name, cls.name)) else: raise # re-raise exception if a different error occurred
def switch(self, dest, url, rev_options): call_subprocess([self.cmd, 'switch'] + rev_options + [url, dest])
def get_revision(self, location): revision = call_subprocess([self.cmd, 'revno'], show_stdout=False, cwd=location) return revision.splitlines()[-1]
def update(self, dest, rev_options): call_subprocess([self.cmd, 'pull', '-q'] + rev_options, cwd=dest)
def update(self, dest, rev_options): call_subprocess(['hg', 'pull', '-q'], cwd=dest) call_subprocess( ['hg', 'update', '-q'] + rev_options, cwd=dest)
def update(self, dest, rev_options): call_subprocess([self.cmd, 'update'] + rev_options + [dest])
def switch(self, dest, url, rev_options): call_subprocess([self.cmd, 'switch', url], cwd=dest)
def update(self, dest, rev_options): call_subprocess([self.cmd, 'fetch', '-q'], cwd=dest) call_subprocess( [self.cmd, 'checkout', '-q', '-f'] + rev_options, cwd=dest)
def _get_revision_from_rev_parse(self, name, location): return call_subprocess([self.cmd, 'rev-parse', name], show_stdout=False, cwd=location)
def _get_all_tag_names(self, location): return call_subprocess([self.cmd, 'tag', '-l'], show_stdout=False, raise_on_returncode=False, cwd=location)
def install( self, install_options, # type: List[str] global_options=None, # type: Optional[Sequence[str]] root=None, # type: Optional[str] home=None, # type: Optional[str] prefix=None, # type: Optional[str] warn_script_location=True, # type: bool use_user_site=False, # type: bool pycompile=True # type: bool ): # type: (...) -> None global_options = global_options if global_options is not None else [] if self.editable: self.install_editable( install_options, global_options, prefix=prefix, ) return if self.is_wheel: version = wheel.wheel_version(self.source_dir) wheel.check_compatibility(version, self.name) self.move_wheel_files( self.source_dir, root=root, prefix=prefix, home=home, warn_script_location=warn_script_location, use_user_site=use_user_site, pycompile=pycompile, ) self.install_succeeded = True return # Extend the list of global and install options passed on to # the setup.py call with the ones from the requirements file. # Options specified in requirements file override those # specified on the command line, since the last option given # to setup.py is the one that is used. global_options = list(global_options) + \ self.options.get('global_options', []) install_options = list(install_options) + \ self.options.get('install_options', []) if self.isolated: # https://github.com/python/mypy/issues/1174 global_options = global_options + ["--no-user-cfg"] # type: ignore with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') install_args = self.get_install_args( global_options, record_filename, root, prefix, pycompile, ) msg = 'Running setup.py install for %s' % (self.name, ) with open_spinner(msg) as spinner: with indent_log(): with self.build_env: call_subprocess( install_args + install_options, cwd=self.setup_py_dir, show_stdout=False, spinner=spinner, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True def prepend_root(path): if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: logger.warning( 'Could not find .egg-info directory in install record' ' for %s', self, ) # FIXME: put the record somewhere # FIXME: should this be an error? return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( os.path.relpath(prepend_root(filename), egg_info_dir)) new_lines.sort() ensure_dir(egg_info_dir) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n')
def switch(self, dest, url, rev_options): repo_config = os.path.join(dest, self.dirname, 'hgrc') config = ConfigParser.SafeConfigParser() try: config.read(repo_config) config.set('paths', 'default', url) config_file = open(repo_config, 'w') config.write(config_file) config_file.close() except (OSError, ConfigParser.NoSectionError), e: logger.warn( 'Could not switch Mercurial repository to %s: %s' % (url, e)) else: call_subprocess(['hg', 'update', '-q'] + rev_options, cwd=dest) def update(self, dest, rev_options): call_subprocess(['hg', 'pull', '-q'], cwd=dest) call_subprocess( ['hg', 'update', '-q'] + rev_options, cwd=dest) def obtain(self, dest): url, rev = self.get_url_rev() if rev: rev_options = [rev] rev_display = ' (to revision %s)' % rev else: rev_options = [] rev_display = '' if self.check_destination(dest, url, rev_options, rev_display):
def get_revision_hash(self, location): current_rev_hash = call_subprocess( [self.cmd, 'parents', '--template={node}'], show_stdout=False, cwd=location).strip() return current_rev_hash
def get_revision(self, location): current_rev = call_subprocess([self.cmd, 'rev-parse', 'HEAD'], show_stdout=False, cwd=location) return current_rev.strip()
def switch(self, dest, url, rev_options): repo_config = os.path.join(dest, self.dirname, 'hgrc') config = ConfigParser.SafeConfigParser() try: config.read(repo_config) config.set('paths', 'default', url) config_file = open(repo_config, 'w') config.write(config_file) config_file.close() except (OSError, ConfigParser.NoSectionError), e: logger.warn( 'Could not switch Mercurial repository to %s: %s' % (url, e)) else: call_subprocess([self.cmd, 'update', '-q'] + rev_options, cwd=dest) def update(self, dest, rev_options): call_subprocess([self.cmd, 'pull', '-q'], cwd=dest) call_subprocess( [self.cmd, 'update', '-q'] + rev_options, cwd=dest) def obtain(self, dest): url, rev = self.get_url_rev() if rev: rev_options = [rev] rev_display = ' (to revision %s)' % rev else: rev_options = [] rev_display = '' if self.check_destination(dest, url, rev_options, rev_display):
def get_revision(self, location): current_revision = call_subprocess( ['hg', 'parents', '--template={rev}'], show_stdout=False, cwd=location).strip() return current_revision
def get_url(self, location): url = call_subprocess([self.cmd, 'config', 'remote.origin.url'], show_stdout=False, cwd=location) return url.strip()
def _get_all_branch_names(self, location): remote_branches = call_subprocess([self.cmd, 'branch', '-r'], show_stdout=False, cwd=location) local_branches = call_subprocess([self.cmd, 'branch', '-l'], show_stdout=False, cwd=location) return remote_branches + local_branches
def switch(self, dest, url, rev_options): call_subprocess([self.cmd, 'config', 'remote.origin.url', url], cwd=dest) call_subprocess([self.cmd, 'checkout', '-q'] + rev_options, cwd=dest)
def update(self, dest, rev_options): # First fetch changes from the default remote call_subprocess([self.cmd, 'fetch', '-q'], cwd=dest) # Then reset to wanted revision (maby even origin/master) call_subprocess([self.cmd, 'reset', '--hard', '-q'] + rev_options, cwd=dest)