def checkout(self, url, version='', verbose=False,
shallow=False, timeout=None):
"""
untars tar at url to self.path.
If version was given, only the subdirectory 'version' of the
tar will end up in self.path. Also creates a file next to the
checkout named *.tar which is a yaml file listing origin url
and version arguments.
"""
if not ensure_dir_notexists(self.get_path()):
self.logger.error("Can't remove %s" % self.get_path())
return False
tempdir = None
result = False
try:
tempdir = tempfile.mkdtemp()
if os.path.isfile(url):
filename = url
else:
(filename, _) = urlretrieve_netrc(url)
# print "filename", filename
temp_tarfile = tarfile.open(filename, 'r:*')
members = None # means all members in extractall
if version == '' or version is None:
subdir = tempdir
self.logger.warn("No tar subdirectory chosen via the 'version' argument for url: %s" % url)
else:
# getmembers lists all files contained in tar with
# relative path
subdirs = []
members = []
for m in temp_tarfile.getmembers():
if m.name.startswith(version + '/'):
members.append(m)
if m.name.split('/')[0] not in subdirs:
subdirs.append(m.name.split('/')[0])
if not members:
raise VcsError("%s is not a subdirectory with contents in members %s" % (version, subdirs))
subdir = os.path.join(tempdir, version)
temp_tarfile.extractall(path=tempdir, members=members)
if not os.path.isdir(subdir):
raise VcsError("%s is not a subdirectory\n" % subdir)
try:
# os.makedirs(os.path.dirname(self._path))
shutil.move(subdir, self._path)
except Exception as ex:
raise VcsError("%s failed to move %s to %s" % (ex, subdir, self._path))
metadata = yaml.dump({'url': url, 'version': version})
with open(self.metadata_path, 'w') as mdat:
mdat.write(metadata)
result = True
except Exception as exc:
self.logger.error("Tarball download unpack failed: %s" % str(exc))
finally:
if tempdir is not None and os.path.exists(tempdir):
rmtree(tempdir)
return result
def _get_bzr_version():
"""Looks up bzr version by calling bzr --version.
:raises: VcsError if bzr is not installed"""
try:
value, output, _ = run_shell_command('bzr --version',
shell=True,
us_env=True)
if value == 0 and output is not None and len(output.splitlines()) > 0:
version = output.splitlines()[0]
else:
raise VcsError("bzr --version returned %s, maybe bzr is not installed" % value)
except VcsError as e:
raise VcsError("Coud not determine whether bzr is installed: %s" % e)
return version
def _get_hg_version():
"""Looks up hg version by calling hg --version.
:raises: VcsError if hg is not installed"""
try:
value, output, _ = run_shell_command('hg --version',
shell=True,
us_env=True)
if value == 0 and output is not None and len(output.splitlines()) > 0:
version = output.splitlines()[0]
else:
raise VcsError(
"hg --version returned %s, output '%s', maybe hg is not installed"
% (value, output))
except VcsError as e:
raise VcsError("Could not determine whether hg is installed %s" % e)
return version
def _get_svn_version():
"""Looks up svn version by calling svn --version.
:raises: VcsError if svn is not installed"""
try:
# SVN commands produce differently formatted output for french locale
value, output, _ = run_shell_command('svn --version',
shell=True,
us_env=True)
if value == 0 and output is not None and len(output.splitlines()) > 0:
version = output.splitlines()[0]
else:
raise VcsError("svn --version returned " +
"%s maybe svn is not installed" % value)
except VcsError as exc:
raise VcsError("Could not determine whether svn is installed: " +
str(exc))
return version
def _get_git_version():
"""Looks up git version by calling git --version.
:raises: VcsError if git is not installed or returns
something unexpected"""
try:
cmd = 'git --version'
value, version, _ = run_shell_command(cmd, shell=True)
if value != 0:
raise VcsError("git --version returned %s, maybe git is not installed" % (value))
prefix = 'git version '
if version is not None and version.startswith(prefix):
version = version[len(prefix):].strip()
else:
raise VcsError("git --version returned invalid string: '%s'" % version)
except VcsError as exc:
raise VcsError("Could not determine whether git is installed: %s" % exc)
return version
def _get_file(self, _repo_type, repo_url, version, filename):
""" Fetch the file specificed by filename relative to the root of the repository"""
name = simplify_repo_name(repo_url)
repo_path = os.path.join(self._cache_location, name)
client = GitClient(repo_path) # using git only
updated = False
if client.path_exists():
if client.get_url() == repo_url:
if not self._skip_update:
logging.disable(logging.WARNING)
updated = client.update(version, force_fetch=True)
logging.disable(logging.NOTSET)
else:
try: # catch exception which can be caused by calling internal API
logging.disable(logging.WARNING)
updated = client._do_update(version)
logging.disable(logging.NOTSET)
except GitError:
updated = False
if not updated:
shutil.rmtree(repo_path)
if not updated:
logging.disable(logging.WARNING)
updated = client.checkout(repo_url, version)
logging.disable(logging.NOTSET)
if not updated:
raise VcsError(
"Impossible to update/checkout repo '%s' with version '%s'." %
(repo_url, version))
full_filename = os.path.join(repo_path, filename)
if not os.path.exists(full_filename):
raise VcsError(
"Requested file '%s' missing from repo '%s' version '%s' (viewed at version '%s'). It was expected at: %s"
% (filename, repo_url, version, client.get_version(),
full_filename))
return full_filename
def sanitized(arg):
"""
makes sure a composed command to be executed via shell was not injected.
A composed command would be like "ls %s"%foo.
In this example, foo could be "; rm -rf *"
sanitized raises an Error when it detects such an attempt
:raises VcsError: on injection attempts
"""
if arg is None or arg.strip() == '':
return ''
arg = str(arg.strip('"').strip())
safe_arg = '"%s"' % arg
# this also detects some false positives, like bar"";foo
if '"' in arg:
if (len(shlex.split(safe_arg, False, False)) != 1):
raise VcsError("Shell injection attempt detected: >%s< = %s" %
(arg, shlex.split(safe_arg, False, False)))
return safe_arg
def get_status(self, basepath=None, untracked=False):
response = None
if basepath is None:
basepath = self._path
if self.path_exists():
rel_path = normalized_rel_path(self._path, basepath)
# protect against shell injection
command = "hg status %(path)s --repository %(path)s" % {
'path': sanitized(rel_path)
}
if not untracked:
command += " -mard"
_, response, _ = run_shell_command(command,
shell=True,
cwd=basepath)
if response is not None:
if response.startswith("abort"):
raise VcsError("Probable Bug; Could not call %s, cwd=%s" %
(command, basepath))
if len(response) > 0 and response[-1] != '\n':
response += '\n'
return response
def export_repository(self, version, basepath):
raise VcsError('export repository not implemented for extracted tars')
def run_shell_command(cmd, cwd=None, shell=False, us_env=True,
show_stdout=False, verbose=False, timeout=None,
no_warn=False, no_filter=False):
"""
executes a command and hides the stdout output, loggs stderr
output when command result is not zero. Make sure to sanitize
arguments in the command.
:param cmd: A string to execute.
:param shell: Whether to use os shell.
:param us_env: changes env var LANG before running command, can influence program output
:param show_stdout: show some of the output (except for discarded lines in _discard_line()), ignored if no_filter
:param no_warn: hides warnings
:param verbose: show all output, overrides no_warn, ignored if no_filter
:param timeout: time allocated to the subprocess
:param no_filter: does not wrap stdout, so invoked command prints everything outside our knowledge
this is DANGEROUS, as vulnerable to shell injection.
:returns: ( returncode, stdout, stderr); stdout is None if no_filter==True
:raises: VcsError on OSError
"""
try:
env = copy.copy(os.environ)
if us_env:
env[str("LANG")] = str("en_US.UTF-8")
if no_filter:
# in no_filter mode, we cannot pipe stdin, as this
# causes some prompts to be hidden (e.g. mercurial over
# http)
stdout_target = None
stderr_target = None
else:
stdout_target = subprocess.PIPE
stderr_target = subprocess.PIPE
# additional parameters to Popen when using a timeout
crflags = {}
if timeout is not None:
if hasattr(os.sys, 'winver'):
crflags['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
else:
crflags['preexec_fn'] = os.setsid
proc = subprocess.Popen(cmd,
shell=shell,
cwd=cwd,
stdout=stdout_target,
stderr=stderr_target,
env=env,
**crflags)
# using a queue to enable usage in a separate thread
q = Queue()
if timeout is None:
_read_shell_output(proc, no_filter, verbose, show_stdout, q)
else:
t = threading.Thread(target=_read_shell_output,
args=[proc, no_filter, verbose, show_stdout, q])
t.start()
t.join(timeout)
if t.isAlive():
if hasattr(os.sys, 'winver'):
os.kill(proc.pid, signal.CTRL_BREAK_EVENT)
else:
os.killpg(proc.pid, signal.SIGTERM)
t.join()
(stdout, stderr) = q.get()
stdout_buf = q.get()
stderr_buf = q.get()
if stdout is not None:
stdout_buf.append(stdout.decode('utf-8'))
stdout = "\n".join(stdout_buf)
if stderr is not None:
stderr_buf.append(stderr.decode('utf-8'))
stderr = "\n".join(stderr_buf)
message = None
if proc.returncode != 0 and stderr is not None and stderr != '':
logger = logging.getLogger('vcstools')
message = "Command failed: '%s'" % (cmd)
if cwd is not None:
message += "\n run at: '%s'" % (cwd)
message += "\n errcode: %s:\n%s" % (proc.returncode, stderr)
if not no_warn:
logger.warn(message)
result = stdout
if result is not None:
result = result.rstrip()
return (proc.returncode, result, message)
except OSError as ose:
logger = logging.getLogger('vcstools')
message = "Command failed with OSError. '%s' <%s, %s>:\n%s" % (cmd, shell, cwd, ose)
logger.error(message)
raise VcsError(message)
