def checkout(self, url, version='', verbose=False, shallow=False, timeout=None): """ untars tar at url to self.path. If version was given, only the subdirectory 'version' of the tar will end up in self.path. Also creates a file next to the checkout named *.tar which is a yaml file listing origin url and version arguments. """ if not ensure_dir_notexists(self.get_path()): self.logger.error("Can't remove %s" % self.get_path()) return False tempdir = None result = False try: tempdir = tempfile.mkdtemp() if os.path.isfile(url): filename = url else: (filename, _) = urlretrieve_netrc(url) # print "filename", filename temp_tarfile = tarfile.open(filename, 'r:*') members = None # means all members in extractall if version == '' or version is None: subdir = tempdir self.logger.warn("No tar subdirectory chosen via the 'version' argument for url: %s" % url) else: # getmembers lists all files contained in tar with # relative path subdirs = [] members = [] for m in temp_tarfile.getmembers(): if m.name.startswith(version + '/'): members.append(m) if m.name.split('/')[0] not in subdirs: subdirs.append(m.name.split('/')[0]) if not members: raise VcsError("%s is not a subdirectory with contents in members %s" % (version, subdirs)) subdir = os.path.join(tempdir, version) temp_tarfile.extractall(path=tempdir, members=members) if not os.path.isdir(subdir): raise VcsError("%s is not a subdirectory\n" % subdir) try: # os.makedirs(os.path.dirname(self._path)) shutil.move(subdir, self._path) except Exception as ex: raise VcsError("%s failed to move %s to %s" % (ex, subdir, self._path)) metadata = yaml.dump({'url': url, 'version': version}) with open(self.metadata_path, 'w') as mdat: mdat.write(metadata) result = True except Exception as exc: self.logger.error("Tarball download unpack failed: %s" % str(exc)) finally: if tempdir is not None and os.path.exists(tempdir): rmtree(tempdir) return result
def _get_bzr_version(): """Looks up bzr version by calling bzr --version. :raises: VcsError if bzr is not installed""" try: value, output, _ = run_shell_command('bzr --version', shell=True, us_env=True) if value == 0 and output is not None and len(output.splitlines()) > 0: version = output.splitlines()[0] else: raise VcsError("bzr --version returned %s, maybe bzr is not installed" % value) except VcsError as e: raise VcsError("Coud not determine whether bzr is installed: %s" % e) return version
def _get_hg_version(): """Looks up hg version by calling hg --version. :raises: VcsError if hg is not installed""" try: value, output, _ = run_shell_command('hg --version', shell=True, us_env=True) if value == 0 and output is not None and len(output.splitlines()) > 0: version = output.splitlines()[0] else: raise VcsError( "hg --version returned %s, output '%s', maybe hg is not installed" % (value, output)) except VcsError as e: raise VcsError("Could not determine whether hg is installed %s" % e) return version
def _get_svn_version(): """Looks up svn version by calling svn --version. :raises: VcsError if svn is not installed""" try: # SVN commands produce differently formatted output for french locale value, output, _ = run_shell_command('svn --version', shell=True, us_env=True) if value == 0 and output is not None and len(output.splitlines()) > 0: version = output.splitlines()[0] else: raise VcsError("svn --version returned " + "%s maybe svn is not installed" % value) except VcsError as exc: raise VcsError("Could not determine whether svn is installed: " + str(exc)) return version
def _get_git_version(): """Looks up git version by calling git --version. :raises: VcsError if git is not installed or returns something unexpected""" try: cmd = 'git --version' value, version, _ = run_shell_command(cmd, shell=True) if value != 0: raise VcsError("git --version returned %s, maybe git is not installed" % (value)) prefix = 'git version ' if version is not None and version.startswith(prefix): version = version[len(prefix):].strip() else: raise VcsError("git --version returned invalid string: '%s'" % version) except VcsError as exc: raise VcsError("Could not determine whether git is installed: %s" % exc) return version
def _get_file(self, _repo_type, repo_url, version, filename): """ Fetch the file specificed by filename relative to the root of the repository""" name = simplify_repo_name(repo_url) repo_path = os.path.join(self._cache_location, name) client = GitClient(repo_path) # using git only updated = False if client.path_exists(): if client.get_url() == repo_url: if not self._skip_update: logging.disable(logging.WARNING) updated = client.update(version, force_fetch=True) logging.disable(logging.NOTSET) else: try: # catch exception which can be caused by calling internal API logging.disable(logging.WARNING) updated = client._do_update(version) logging.disable(logging.NOTSET) except GitError: updated = False if not updated: shutil.rmtree(repo_path) if not updated: logging.disable(logging.WARNING) updated = client.checkout(repo_url, version) logging.disable(logging.NOTSET) if not updated: raise VcsError( "Impossible to update/checkout repo '%s' with version '%s'." % (repo_url, version)) full_filename = os.path.join(repo_path, filename) if not os.path.exists(full_filename): raise VcsError( "Requested file '%s' missing from repo '%s' version '%s' (viewed at version '%s'). It was expected at: %s" % (filename, repo_url, version, client.get_version(), full_filename)) return full_filename
def sanitized(arg): """ makes sure a composed command to be executed via shell was not injected. A composed command would be like "ls %s"%foo. In this example, foo could be "; rm -rf *" sanitized raises an Error when it detects such an attempt :raises VcsError: on injection attempts """ if arg is None or arg.strip() == '': return '' arg = str(arg.strip('"').strip()) safe_arg = '"%s"' % arg # this also detects some false positives, like bar"";foo if '"' in arg: if (len(shlex.split(safe_arg, False, False)) != 1): raise VcsError("Shell injection attempt detected: >%s< = %s" % (arg, shlex.split(safe_arg, False, False))) return safe_arg
def get_status(self, basepath=None, untracked=False): response = None if basepath is None: basepath = self._path if self.path_exists(): rel_path = normalized_rel_path(self._path, basepath) # protect against shell injection command = "hg status %(path)s --repository %(path)s" % { 'path': sanitized(rel_path) } if not untracked: command += " -mard" _, response, _ = run_shell_command(command, shell=True, cwd=basepath) if response is not None: if response.startswith("abort"): raise VcsError("Probable Bug; Could not call %s, cwd=%s" % (command, basepath)) if len(response) > 0 and response[-1] != '\n': response += '\n' return response
def export_repository(self, version, basepath): raise VcsError('export repository not implemented for extracted tars')
def run_shell_command(cmd, cwd=None, shell=False, us_env=True, show_stdout=False, verbose=False, timeout=None, no_warn=False, no_filter=False): """ executes a command and hides the stdout output, loggs stderr output when command result is not zero. Make sure to sanitize arguments in the command. :param cmd: A string to execute. :param shell: Whether to use os shell. :param us_env: changes env var LANG before running command, can influence program output :param show_stdout: show some of the output (except for discarded lines in _discard_line()), ignored if no_filter :param no_warn: hides warnings :param verbose: show all output, overrides no_warn, ignored if no_filter :param timeout: time allocated to the subprocess :param no_filter: does not wrap stdout, so invoked command prints everything outside our knowledge this is DANGEROUS, as vulnerable to shell injection. :returns: ( returncode, stdout, stderr); stdout is None if no_filter==True :raises: VcsError on OSError """ try: env = copy.copy(os.environ) if us_env: env[str("LANG")] = str("en_US.UTF-8") if no_filter: # in no_filter mode, we cannot pipe stdin, as this # causes some prompts to be hidden (e.g. mercurial over # http) stdout_target = None stderr_target = None else: stdout_target = subprocess.PIPE stderr_target = subprocess.PIPE # additional parameters to Popen when using a timeout crflags = {} if timeout is not None: if hasattr(os.sys, 'winver'): crflags['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP else: crflags['preexec_fn'] = os.setsid proc = subprocess.Popen(cmd, shell=shell, cwd=cwd, stdout=stdout_target, stderr=stderr_target, env=env, **crflags) # using a queue to enable usage in a separate thread q = Queue() if timeout is None: _read_shell_output(proc, no_filter, verbose, show_stdout, q) else: t = threading.Thread(target=_read_shell_output, args=[proc, no_filter, verbose, show_stdout, q]) t.start() t.join(timeout) if t.isAlive(): if hasattr(os.sys, 'winver'): os.kill(proc.pid, signal.CTRL_BREAK_EVENT) else: os.killpg(proc.pid, signal.SIGTERM) t.join() (stdout, stderr) = q.get() stdout_buf = q.get() stderr_buf = q.get() if stdout is not None: stdout_buf.append(stdout.decode('utf-8')) stdout = "\n".join(stdout_buf) if stderr is not None: stderr_buf.append(stderr.decode('utf-8')) stderr = "\n".join(stderr_buf) message = None if proc.returncode != 0 and stderr is not None and stderr != '': logger = logging.getLogger('vcstools') message = "Command failed: '%s'" % (cmd) if cwd is not None: message += "\n run at: '%s'" % (cwd) message += "\n errcode: %s:\n%s" % (proc.returncode, stderr) if not no_warn: logger.warn(message) result = stdout if result is not None: result = result.rstrip() return (proc.returncode, result, message) except OSError as ose: logger = logging.getLogger('vcstools') message = "Command failed with OSError. '%s' <%s, %s>:\n%s" % (cmd, shell, cwd, ose) logger.error(message) raise VcsError(message)