def cmd(command, echo=False, env=None, show_output=True, critical=False):
new_env = _cmd_handle_env(env)
command = _cmd_handle_args(command)
if echo:
log.info('$ ' + (' '.join(command)))
if show_output:
return subprocess.call(command, env=new_env, shell=False) == 0
output = ''
try:
output = subprocess.check_output(command,
env=new_env,
stderr=subprocess.STDOUT)
return True
except CalledProcessError as cpe:
log.error(cpe.output)
if critical:
raise cpe
log.error(cpe)
return False
except Exception as e:
log.error(e)
log.error(output)
if critical:
raise e
log.error(e)
return False
def GetDpkgShlibs(files):
deps = {}
stdout, stderr = cmd_output(['perl', os.path.join(scripts_dir, 'dpkg-dump-shpkgs.pl')] + files, critical=True)
if stdout or stderr:
for line in (stdout + stderr).split('\n'):
line = line.strip()
if line == '':
continue
# dpkg-dump-shpkgs.pl: warning: binaries to analyze should already
# be installed in their package's directory
if 'dpkg-dump-shpkgs.pl:' in line:
(_, msgtype, msg) = [x.strip() for x in line.split(':')]
if msg == 'binaries to analyze should already be installed in their package\'s directory':
continue
if msgtype == 'warning':
log.warning(msg)
elif msgtype == 'error':
log.error(msg)
continue
elif line.startswith('shlibs:'):
# shlibs:Depends=libboost-context1.55.0,
# libboost-filesystem1.55.0, libboost-program-options1.55.0,
# ...
lc = line.split('=', 1)
assert len(lc) == 2
assert not lc[0][7:].startswith(':')
deps[lc[0][7:]] = [x.strip() for x in lc[1].split(',')]
else:
log.warning('UNHANDLED: %s', line)
return deps
def GetDpkgShlibs(files):
deps = {}
stdout, stderr = cmd_output(
['perl', os.path.join(scripts_dir, 'dpkg-dump-shpkgs.pl')] + files,
critical=True)
if stdout or stderr:
for line in (stdout + stderr).decode('utf-8').split('\n'):
line = line.strip()
if line == '':
continue
# dpkg-dump-shpkgs.pl: warning: binaries to analyze should already
# be installed in their package's directory
if 'dpkg-dump-shpkgs.pl:' in line:
(_, msgtype, msg) = [x.strip() for x in line.split(':')]
if msg == 'binaries to analyze should already be installed in their package\'s directory':
continue
if msgtype == 'warning':
log.warning(msg)
elif msgtype == 'error':
log.error(msg)
continue
elif line.startswith('shlibs:'):
# shlibs:Depends=libboost-context1.55.0,
# libboost-filesystem1.55.0, libboost-program-options1.55.0,
# ...
lc = line.split('=', 1)
assert len(lc) == 2
assert not lc[0][7:].startswith(':')
deps[lc[0][7:]] = [x.strip() for x in lc[1].split(',')]
else:
log.warning('UNHANDLED: %s', line)
return deps
def cmd(command, echo=False, env=None, show_output=True, critical=False):
new_env = _cmd_handle_env(env)
command = _cmd_handle_args(command)
if echo:
log.info('$ ' + (' '.join(command)))
if show_output:
return subprocess.call(command, env=new_env, shell=False) == 0
output = ''
try:
output = subprocess.check_output(command, env=new_env, stderr=subprocess.STDOUT)
return True
except CalledProcessError as cpe:
log.error(cpe.output)
if critical:
raise cpe
log.error(cpe)
return False
except Exception as e:
log.error(e)
log.error(output)
if critical:
raise e
log.error(e)
return False
def DeserializeXML(cls, element):
assert element.tag == 'Project'
proj = Project()
proj.getAttr(element, 'frameworkVersion', required=True)
proj.getAttr(element, 'name', required=True)
proj.getAttr(element, 'type', required=True)
proj.getAttr(element, 'path', required=True)
proj.getAttr(element, 'rootNamespace')
for child in element:
if not etree.iselement(child) or child.tag is etree.Comment:
continue
if child.tag == 'Configuration':
proj.configurations += [Configuration.DeserializeXML(child)]
elif child.tag == 'ReferencePath':
proj.referencePaths += [child.text]
elif child.tag == 'Files':
for filedata in child:
if not etree.iselement(
filedata) or filedata.tag is etree.Comment:
continue
proj.files += [File.DeserializeXML(filedata)]
elif child.tag == 'Reference':
proj.references += [Reference.DeserializeXML(child)]
else:
log.error('!!! Unknown project tag child {}'.format(child.tag))
return proj
def cmd_out(command: Union[List[str], str],
echo=False,
env=None,
critical=False,
globbify=True,
encoding: str = 'utf-8') -> str:
'''
:returns str: stderr and stdout, piped into one string and decoded as UTF-8.
'''
new_env = _cmd_handle_env(env)
command = _cmd_handle_args(command, globbify)
if echo:
log.info('$ ' + _args2str(command))
try:
p = subprocess.Popen(command,
env=new_env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True)
return p.stdout.read().decode('utf-8')
except Exception as e:
log.error(repr(command))
if critical:
raise e
log.error(e)
return None
def fix_encoding(filename, encoding='utf-8-sig'):
#log.info('chardet guesses: {}'.format(encoding))
if encoding in ('utf-8-sig', 'cp1252'):
with codecs.open(filename, 'r', encoding=encoding) as inf:
with codecs.open(filename + '.utf8', 'w',
encoding='utf-8-sig') as outf:
for line in ftfy.fix_file(inf,
fix_entities=False,
fix_latin_ligatures=False,
fix_character_width=False,
uncurl_quotes=False):
outf.write(line)
# This is here because Windows 10 was locking files randomly.
attempt = 0
while attempt < 5:
attempt += 1
try:
if os.path.isfile(filename):
os.remove(filename)
break
except PermissionError:
log.error("[%d/%d] Failed to delete %s, trying again in 1s.",
attempt, 5, filename)
time.sleep(0.5)
shutil.move(filename + '.utf8', filename)
return encoding
def cmd_daemonize(command,
echo=False,
env=None,
critical=False,
globbify=True):
new_env = _cmd_handle_env(env)
command = _cmd_handle_args(command, globbify)
if echo:
log.info('& ' + _args2str(command))
try:
if platform.system() == 'Windows':
# HACK
batch = os.tmpnam() + '.bat'
with open(batch, 'w') as b:
b.write(' '.join(command))
os.startfile(batch)
else:
subprocess.Popen(command, env=new_env)
return True
except Exception as e:
log.error(repr(command))
if critical:
raise e
log.error(e)
return False
def UpdateRemotes(self):
stdout, stderr = cmd_output(['git', 'remote', 'show'],
echo=not self.quiet)
for line in (stdout + stderr).split('\n'):
line = line.strip()
if line == '':
continue
if line.startswith('fatal:'):
log.error('[git] ' + line)
return False
self.remotes[line] = self._getRemoteInfo(line)
return True
def GetBranch(cls, quiet=True):
try:
cmd = ["git", "symbolic-ref", "--short", 'HEAD']
#stderr, stdout = cmd_output(["git", "rev-parse", "--abbrev-ref", 'HEAD', '--'], echo=not quiet, critical=True)
stderr, stdout = cmd_output(cmd, echo=not quiet, critical=True)
o = (stderr + stdout).decode('utf-8').strip()
if cls.OutputIsFatal(o):
log.error(_args2str(cmd))
return None
return o.strip()
except Exception as e:
print(e)
pass
return None
def Load(self,
filename,
merge=False,
defaults=None,
variables={},
verbose=False):
lh = NullIndenter()
if self.actual_filename is None:
self.actual_filename = filename if self.template_dir is None else os.path.join(
self.template_dir, filename)
if verbose:
lh = log.info("Loading %s...", filename)
with lh:
if not self.skip_defaults and not os.path.isfile(filename):
if defaults is None:
if verbose: log.error('Failed to load %s.', filename)
return False
else:
if verbose: log.warn('File not found, loading defaults.')
ensureDirExists(os.path.dirname(filename))
self.dump_to_file(filename, defaults)
rendered = ''
template: jinja2.Template = None
#print(repr(self.template_dir))
try:
if self.template_dir is not None:
template = self.environment.get_template(
os.path.basename(filename)
if self.template_dir is None else filename)
else:
with open(filename) as f:
template = self.environment.from_string(f.read())
rendered = template.render(variables)
except jinja2.exceptions.TemplateNotFound as tnf:
if verbose:
log.warn(
'Jinja2 failed to load %s (TemplateNotFound). Failing over to plain string.',
filename)
log.exception(tnf)
with codecs.open(filename, 'r', encoding=self.encoding) as f:
rendered = f.read()
#print(rendered)
newcfg = self.load_from_string(rendered)
if merge:
self.cfg = dict_merge(self.cfg, newcfg)
else:
self.cfg = newcfg
return True
def writeGivenIndentData(writefunc,
lines,
writeIndentedLine,
offsets=0,
override_writeindented=None):
indent = 0
indentOffsets = ['manuallyoffset'] * offsets
def innerwriteindented(curindent, line):
writefunc(' ' * (curindent * 4) + line + '\n')
def writeindented(line, indentOffset, offset=0):
innerwriteindented((indent + indentOffset), line)
if override_writeindented:
innerwriteindented = override_writeindented
nLines = len(lines)
lastLineIndented = False
for i in range(nLines):
diff, currentIndent, line = lines[i]
ndiff = None
ndiffidx = 1
while ndiff is None:
if i + ndiffidx < nLines:
ndiff, _, _ = lines[i + ndiffidx]
ndiffidx += 1
else:
break
if diff is None:
line = line.strip()
if line.startswith('#comment'):
continue
if line.startswith('#startblock'):
statement = line[12:].strip()
if ndiff == 1:
indentOffsets.append(statement)
writeindented(statement, len(indentOffsets))
if ndiff < 1:
indentOffsets.append(statement)
continue
if line == '#endblock':
statement = indentOffsets.pop()
writeindented('# END {}'.format(statement), len(indentOffsets))
continue
log.error('UNKNOWN TEMPLATE ENGINE COMMAND: ' + line.strip())
continue
indent = max(indent + diff, 0)
lastLineIndented = writeIndentedLine(indent, diff, ndiff, line,
len(indentOffsets), writeindented)
def Pull(self,
remote='origin',
branch='HEAD',
commit=None,
tag=None,
cleanup=False):
if branch == 'HEAD':
branch = self.remotes[remote].head_branch
if self.submodule:
log.error('Submodules should not call Pull!')
return
if not os.path.isdir(self.path):
cmd(['git', 'clone', self.remotes[remote].fetch_uri, self.path],
echo=not self.quiet or self.noisy_clone,
critical=True,
show_output=not self.quiet or self.noisy_clone,
env=self.noPasswordEnv)
with Chdir(self.path, quiet=self.quiet):
if cleanup:
cmd(['git', 'clean', '-fdx'],
echo=not self.quiet,
critical=True)
cmd(['git', 'reset', '--hard'],
echo=not self.quiet,
critical=True)
if self.current_branch != branch:
ref = 'remotes/{}/{}'.format(remote, branch)
cmd(['git', 'checkout', '-B', branch, ref, '--'],
echo=not self.quiet,
critical=True)
if tag is not None:
commit = self._resolveTagNoChdir(tag)
if commit is not None:
cmd(['git', 'checkout', commit],
echo=not self.quiet,
critical=True)
else:
if self.current_commit != self.remote_commit:
cmd([
'git', 'reset', '--hard', '{}/{}'.format(
remote, branch)
],
echo=not self.quiet,
critical=True)
if self.UsesLFS():
log.info('git-lfs detected!')
cmd(['git', 'lfs', 'pull'], echo=not self.quiet, critical=True)
return True
def cmd_output(command, echo=False, env=None, critical=False):
'''
:returns List[2]: (stdout,stderr)
'''
new_env = _cmd_handle_env(env)
command = _cmd_handle_args(command)
if echo:
log.info('$ ' + (' '.join(command)))
try:
return subprocess.Popen(command, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
except Exception as e:
log.error(repr(command))
if critical:
raise e
log.error(e)
return False
def _resolveTagNoChdir(self, tag, quiet=None):
if quiet is None:
quiet = self.quiet
ret = cmd_output(
['git', 'rev-list', '-n', '1', 'refs/tags/{}'.format(tag)],
echo=not quiet,
env=self.noPasswordEnv)
if not ret:
return None
stdout, stderr = ret
for line in (stdout + stderr).decode('utf-8').split('\n'):
line = line.strip()
if line == '':
continue
if line.startswith('fatal:'):
log.error('[git] ' + line)
return None
return line.strip()
return None
def cmd_output(command, echo=False, env=None, critical=False):
'''
:returns List[2]: (stdout,stderr)
'''
new_env = _cmd_handle_env(env)
command = _cmd_handle_args(command)
if echo:
log.info('$ ' + (' '.join(command)))
try:
return subprocess.Popen(command,
env=new_env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
except Exception as e:
log.error(repr(command))
if critical:
raise e
log.error(e)
return False
def Load(self, filename, merge=False, defaults=None, variables={}):
with log.info("Loading %s...", filename):
if not os.path.isfile(filename):
if defaults is None:
log.error('Failed to load %s.', filename)
return False
else:
log.warn('File not found, loading defaults.')
with open(filename, 'w') as f:
yaml.dump(defaults, f, default_flow_style=False)
template = self.environment.get_template(filename)
rendered = template.render(variables)
newcfg = yaml.load(rendered)
if merge:
self.cfg = dict_merge(self.cfg, newcfg)
else:
self.cfg = newcfg
return True
def GetRemoteState(self, remote='origin', branch='master'):
with Chdir(self.path, quiet=self.quiet):
ret = cmd_output(['git', 'fetch', '-q'], echo=not self.quiet)
if not ret:
return False
stdout, stderr = ret
for line in (stdout + stderr).split('\n'):
line = line.strip()
if line == '':
continue
if line.startswith('fatal:'):
log.error('[git] ' + line)
return False
remoteinfo = Git.LSRemote(remote, branch)
if remoteinfo is None:
return False
ref = 'refs/heads/' + branch
if ref in remoteinfo:
self.remote_commit = remoteinfo[ref]
return True
def UpdateRemotes(self, remote=None, quiet=None):
if quiet is None:
quiet = self.quiet
if remote is not None:
self.remotes[remote] = self._syncRemote(remote, quiet=quiet)
return True
stdout, stderr = cmd_output(['git', 'remote', 'show'],
echo=not quiet,
env=self.noPasswordEnv)
for oline in (stdout + stderr).decode('utf-8').split('\n'):
line = oline.strip()
if not quiet or not quiet:
print(oline)
if line == '':
continue
if line.startswith('fatal:'):
log.error('[git] ' + line)
return False
self.remotes[line] = self._syncRemote(line, quiet=quiet)
return True
def cmd(command,
echo=False,
env=None,
show_output=True,
critical=False,
globbify=True,
acceptable_exit_codes=[0]):
new_env = _cmd_handle_env(env)
command = _cmd_handle_args(command, globbify)
if echo:
log.info('$ ' + _args2str(command))
output = ''
try:
if show_output:
code = subprocess.call(command, env=new_env, shell=False)
#print(repr(code))
success = code in acceptable_exit_codes
if critical and not success:
raise CalledProcessError(code, command)
return success
else:
# Using our own customized check_output for acceptable_exit_codes.
output = check_output(command,
env=new_env,
stderr=subprocess.STDOUT,
acceptable_exit_codes=acceptable_exit_codes)
return True
except CalledProcessError as cpe:
log.error(cpe.output)
if critical:
raise cpe
log.error(cpe)
return False
except Exception as e:
log.error(e)
log.error(output)
if critical:
raise e
log.error(e)
return False
def cmd_daemonize(command, echo=False, env=None, critical=False):
new_env = _cmd_handle_env(env)
command = _cmd_handle_args(command)
if echo:
log.info('& ' + ' '.join(command))
try:
if platform.system() == 'Windows':
# HACK
batch = os.tmpnam() + '.bat'
with open(batch, 'w') as b:
b.write(' '.join(command))
os.startfile(batch)
else:
subprocess.Popen(command, env=new_env)
return True
except Exception as e:
log.error(repr(command))
if critical:
raise e
log.error(e)
return False
def DpkgSearchFiles(files):
'''Find packages for a given set of files.'''
stdout, stderr = cmd_output(['dpkg', '--search'] + files, critical=True)
'''
libc6:amd64: /lib/x86_64-linux-gnu/libc-2.19.so
libcap2:amd64: /lib/x86_64-linux-gnu/libcap.so.2
libcap2:amd64: /lib/x86_64-linux-gnu/libcap.so.2.24
libc6:amd64: /lib/x86_64-linux-gnu/libcidn-2.19.so
libc6:amd64: /lib/x86_64-linux-gnu/libcidn.so.1
libcomerr2:amd64: /lib/x86_64-linux-gnu/libcom_err.so.2
libcomerr2:amd64: /lib/x86_64-linux-gnu/libcom_err.so.2.1
libc6:amd64: /lib/x86_64-linux-gnu/libcrypt-2.19.so
libcryptsetup4:amd64: /lib/x86_64-linux-gnu/libcryptsetup.so.4
libcryptsetup4:amd64: /lib/x86_64-linux-gnu/libcryptsetup.so.4.6.0
libc6:amd64: /lib/x86_64-linux-gnu/libcrypt.so.1
libc6:amd64: /lib/x86_64-linux-gnu/libc.so.6
'''
packages = []
if stdout or stderr:
for line in (stdout + stderr).decode('utf-8').split('\n'):
line = line.strip()
if line == '':
continue
chunks = line.split()
# libc6:amd64: /lib/x86_64-linux-gnu/libc.so.6
if len(chunks) == 2:
pkgName = chunks[0][:-1] # Strip ending colon
if pkgName not in packages:
packages += [pkgName]
else:
log.error('UNHANDLED dpkg --search LINE (len == %d): "%s"',
len(chunks), line)
return packages
def DpkgSearchFiles(files):
'''Find packages for a given set of files.'''
stdout, stderr = cmd_output(['dpkg', '--search'] + files, critical=True)
'''
libc6:amd64: /lib/x86_64-linux-gnu/libc-2.19.so
libcap2:amd64: /lib/x86_64-linux-gnu/libcap.so.2
libcap2:amd64: /lib/x86_64-linux-gnu/libcap.so.2.24
libc6:amd64: /lib/x86_64-linux-gnu/libcidn-2.19.so
libc6:amd64: /lib/x86_64-linux-gnu/libcidn.so.1
libcomerr2:amd64: /lib/x86_64-linux-gnu/libcom_err.so.2
libcomerr2:amd64: /lib/x86_64-linux-gnu/libcom_err.so.2.1
libc6:amd64: /lib/x86_64-linux-gnu/libcrypt-2.19.so
libcryptsetup4:amd64: /lib/x86_64-linux-gnu/libcryptsetup.so.4
libcryptsetup4:amd64: /lib/x86_64-linux-gnu/libcryptsetup.so.4.6.0
libc6:amd64: /lib/x86_64-linux-gnu/libcrypt.so.1
libc6:amd64: /lib/x86_64-linux-gnu/libc.so.6
'''
packages = []
if stdout or stderr:
for line in (stdout + stderr).split('\n'):
line = line.strip()
if line == '':
continue
chunks = line.split()
# libc6:amd64: /lib/x86_64-linux-gnu/libc.so.6
if len(chunks) == 2:
pkgName = chunks[0][:-1] # Strip ending colon
if pkgName not in packages:
packages += [pkgName]
else:
log.error('UNHANDLED dpkg --search LINE (len == %d): "%s"', len(chunks), line)
return packages
def GetRemoteState(self, remote='origin', branch='HEAD', quiet=None):
if quiet is None:
quiet = self.quiet
with Chdir(self.path, quiet=self.quiet):
ret = cmd_output(
['git', 'fetch', '-q', '--all', '--prune', '--tags'],
echo=not quiet,
env=self.noPasswordEnv)
if not ret:
return False
stdout, stderr = ret
for line in (stdout + stderr).decode('utf-8').split('\n'):
# if not quiet:
# print(line)
line = line.strip()
if line == '':
continue
if line.startswith('fatal:'):
log.error('[git] ' + line)
return False
for _remote in self.remotes.values():
_remote.fetched = False
self.UpdateRemotes(remote=remote, quiet=quiet)
if branch == 'HEAD':
branch = self.remotes[remote].findBranch(branch)
remoteinfo = Git.LSRemote(remote, branch, quiet=quiet)
if remoteinfo is None:
return False
if branch == 'HEAD':
ref = 'HEAD'
elif '/' not in branch:
ref = 'refs/heads/' + branch
if ref in remoteinfo:
self.remote_commit = remoteinfo[ref]
return True
def cmd_output(command,
echo=False,
env=None,
critical=False,
globbify=True) -> Tuple[bytes, bytes]:
'''
:returns List[2]: (stdout,stderr)
'''
new_env = _cmd_handle_env(env)
command = _cmd_handle_args(command, globbify)
if echo:
log.info('$ ' + _args2str(command))
try:
return subprocess.Popen(command,
env=new_env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
except Exception as e:
log.error(repr(command))
if critical:
raise e
log.error(e)
return False
