def cmd(command, echo=False, env=None, show_output=True, critical=False): new_env = _cmd_handle_env(env) command = _cmd_handle_args(command) if echo: log.info('$ ' + (' '.join(command))) if show_output: return subprocess.call(command, env=new_env, shell=False) == 0 output = '' try: output = subprocess.check_output(command, env=new_env, stderr=subprocess.STDOUT) return True except CalledProcessError as cpe: log.error(cpe.output) if critical: raise cpe log.error(cpe) return False except Exception as e: log.error(e) log.error(output) if critical: raise e log.error(e) return False
def GetDpkgShlibs(files): deps = {} stdout, stderr = cmd_output(['perl', os.path.join(scripts_dir, 'dpkg-dump-shpkgs.pl')] + files, critical=True) if stdout or stderr: for line in (stdout + stderr).split('\n'): line = line.strip() if line == '': continue # dpkg-dump-shpkgs.pl: warning: binaries to analyze should already # be installed in their package's directory if 'dpkg-dump-shpkgs.pl:' in line: (_, msgtype, msg) = [x.strip() for x in line.split(':')] if msg == 'binaries to analyze should already be installed in their package\'s directory': continue if msgtype == 'warning': log.warning(msg) elif msgtype == 'error': log.error(msg) continue elif line.startswith('shlibs:'): # shlibs:Depends=libboost-context1.55.0, # libboost-filesystem1.55.0, libboost-program-options1.55.0, # ... lc = line.split('=', 1) assert len(lc) == 2 assert not lc[0][7:].startswith(':') deps[lc[0][7:]] = [x.strip() for x in lc[1].split(',')] else: log.warning('UNHANDLED: %s', line) return deps
def GetDpkgShlibs(files): deps = {} stdout, stderr = cmd_output( ['perl', os.path.join(scripts_dir, 'dpkg-dump-shpkgs.pl')] + files, critical=True) if stdout or stderr: for line in (stdout + stderr).decode('utf-8').split('\n'): line = line.strip() if line == '': continue # dpkg-dump-shpkgs.pl: warning: binaries to analyze should already # be installed in their package's directory if 'dpkg-dump-shpkgs.pl:' in line: (_, msgtype, msg) = [x.strip() for x in line.split(':')] if msg == 'binaries to analyze should already be installed in their package\'s directory': continue if msgtype == 'warning': log.warning(msg) elif msgtype == 'error': log.error(msg) continue elif line.startswith('shlibs:'): # shlibs:Depends=libboost-context1.55.0, # libboost-filesystem1.55.0, libboost-program-options1.55.0, # ... lc = line.split('=', 1) assert len(lc) == 2 assert not lc[0][7:].startswith(':') deps[lc[0][7:]] = [x.strip() for x in lc[1].split(',')] else: log.warning('UNHANDLED: %s', line) return deps
def DeserializeXML(cls, element): assert element.tag == 'Project' proj = Project() proj.getAttr(element, 'frameworkVersion', required=True) proj.getAttr(element, 'name', required=True) proj.getAttr(element, 'type', required=True) proj.getAttr(element, 'path', required=True) proj.getAttr(element, 'rootNamespace') for child in element: if not etree.iselement(child) or child.tag is etree.Comment: continue if child.tag == 'Configuration': proj.configurations += [Configuration.DeserializeXML(child)] elif child.tag == 'ReferencePath': proj.referencePaths += [child.text] elif child.tag == 'Files': for filedata in child: if not etree.iselement( filedata) or filedata.tag is etree.Comment: continue proj.files += [File.DeserializeXML(filedata)] elif child.tag == 'Reference': proj.references += [Reference.DeserializeXML(child)] else: log.error('!!! Unknown project tag child {}'.format(child.tag)) return proj
def cmd_out(command: Union[List[str], str], echo=False, env=None, critical=False, globbify=True, encoding: str = 'utf-8') -> str: ''' :returns str: stderr and stdout, piped into one string and decoded as UTF-8. ''' new_env = _cmd_handle_env(env) command = _cmd_handle_args(command, globbify) if echo: log.info('$ ' + _args2str(command)) try: p = subprocess.Popen(command, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) return p.stdout.read().decode('utf-8') except Exception as e: log.error(repr(command)) if critical: raise e log.error(e) return None
def fix_encoding(filename, encoding='utf-8-sig'): #log.info('chardet guesses: {}'.format(encoding)) if encoding in ('utf-8-sig', 'cp1252'): with codecs.open(filename, 'r', encoding=encoding) as inf: with codecs.open(filename + '.utf8', 'w', encoding='utf-8-sig') as outf: for line in ftfy.fix_file(inf, fix_entities=False, fix_latin_ligatures=False, fix_character_width=False, uncurl_quotes=False): outf.write(line) # This is here because Windows 10 was locking files randomly. attempt = 0 while attempt < 5: attempt += 1 try: if os.path.isfile(filename): os.remove(filename) break except PermissionError: log.error("[%d/%d] Failed to delete %s, trying again in 1s.", attempt, 5, filename) time.sleep(0.5) shutil.move(filename + '.utf8', filename) return encoding
def cmd_daemonize(command, echo=False, env=None, critical=False, globbify=True): new_env = _cmd_handle_env(env) command = _cmd_handle_args(command, globbify) if echo: log.info('& ' + _args2str(command)) try: if platform.system() == 'Windows': # HACK batch = os.tmpnam() + '.bat' with open(batch, 'w') as b: b.write(' '.join(command)) os.startfile(batch) else: subprocess.Popen(command, env=new_env) return True except Exception as e: log.error(repr(command)) if critical: raise e log.error(e) return False
def UpdateRemotes(self): stdout, stderr = cmd_output(['git', 'remote', 'show'], echo=not self.quiet) for line in (stdout + stderr).split('\n'): line = line.strip() if line == '': continue if line.startswith('fatal:'): log.error('[git] ' + line) return False self.remotes[line] = self._getRemoteInfo(line) return True
def GetBranch(cls, quiet=True): try: cmd = ["git", "symbolic-ref", "--short", 'HEAD'] #stderr, stdout = cmd_output(["git", "rev-parse", "--abbrev-ref", 'HEAD', '--'], echo=not quiet, critical=True) stderr, stdout = cmd_output(cmd, echo=not quiet, critical=True) o = (stderr + stdout).decode('utf-8').strip() if cls.OutputIsFatal(o): log.error(_args2str(cmd)) return None return o.strip() except Exception as e: print(e) pass return None
def Load(self, filename, merge=False, defaults=None, variables={}, verbose=False): lh = NullIndenter() if self.actual_filename is None: self.actual_filename = filename if self.template_dir is None else os.path.join( self.template_dir, filename) if verbose: lh = log.info("Loading %s...", filename) with lh: if not self.skip_defaults and not os.path.isfile(filename): if defaults is None: if verbose: log.error('Failed to load %s.', filename) return False else: if verbose: log.warn('File not found, loading defaults.') ensureDirExists(os.path.dirname(filename)) self.dump_to_file(filename, defaults) rendered = '' template: jinja2.Template = None #print(repr(self.template_dir)) try: if self.template_dir is not None: template = self.environment.get_template( os.path.basename(filename) if self.template_dir is None else filename) else: with open(filename) as f: template = self.environment.from_string(f.read()) rendered = template.render(variables) except jinja2.exceptions.TemplateNotFound as tnf: if verbose: log.warn( 'Jinja2 failed to load %s (TemplateNotFound). Failing over to plain string.', filename) log.exception(tnf) with codecs.open(filename, 'r', encoding=self.encoding) as f: rendered = f.read() #print(rendered) newcfg = self.load_from_string(rendered) if merge: self.cfg = dict_merge(self.cfg, newcfg) else: self.cfg = newcfg return True
def writeGivenIndentData(writefunc, lines, writeIndentedLine, offsets=0, override_writeindented=None): indent = 0 indentOffsets = ['manuallyoffset'] * offsets def innerwriteindented(curindent, line): writefunc(' ' * (curindent * 4) + line + '\n') def writeindented(line, indentOffset, offset=0): innerwriteindented((indent + indentOffset), line) if override_writeindented: innerwriteindented = override_writeindented nLines = len(lines) lastLineIndented = False for i in range(nLines): diff, currentIndent, line = lines[i] ndiff = None ndiffidx = 1 while ndiff is None: if i + ndiffidx < nLines: ndiff, _, _ = lines[i + ndiffidx] ndiffidx += 1 else: break if diff is None: line = line.strip() if line.startswith('#comment'): continue if line.startswith('#startblock'): statement = line[12:].strip() if ndiff == 1: indentOffsets.append(statement) writeindented(statement, len(indentOffsets)) if ndiff < 1: indentOffsets.append(statement) continue if line == '#endblock': statement = indentOffsets.pop() writeindented('# END {}'.format(statement), len(indentOffsets)) continue log.error('UNKNOWN TEMPLATE ENGINE COMMAND: ' + line.strip()) continue indent = max(indent + diff, 0) lastLineIndented = writeIndentedLine(indent, diff, ndiff, line, len(indentOffsets), writeindented)
def Pull(self, remote='origin', branch='HEAD', commit=None, tag=None, cleanup=False): if branch == 'HEAD': branch = self.remotes[remote].head_branch if self.submodule: log.error('Submodules should not call Pull!') return if not os.path.isdir(self.path): cmd(['git', 'clone', self.remotes[remote].fetch_uri, self.path], echo=not self.quiet or self.noisy_clone, critical=True, show_output=not self.quiet or self.noisy_clone, env=self.noPasswordEnv) with Chdir(self.path, quiet=self.quiet): if cleanup: cmd(['git', 'clean', '-fdx'], echo=not self.quiet, critical=True) cmd(['git', 'reset', '--hard'], echo=not self.quiet, critical=True) if self.current_branch != branch: ref = 'remotes/{}/{}'.format(remote, branch) cmd(['git', 'checkout', '-B', branch, ref, '--'], echo=not self.quiet, critical=True) if tag is not None: commit = self._resolveTagNoChdir(tag) if commit is not None: cmd(['git', 'checkout', commit], echo=not self.quiet, critical=True) else: if self.current_commit != self.remote_commit: cmd([ 'git', 'reset', '--hard', '{}/{}'.format( remote, branch) ], echo=not self.quiet, critical=True) if self.UsesLFS(): log.info('git-lfs detected!') cmd(['git', 'lfs', 'pull'], echo=not self.quiet, critical=True) return True
def cmd_output(command, echo=False, env=None, critical=False): ''' :returns List[2]: (stdout,stderr) ''' new_env = _cmd_handle_env(env) command = _cmd_handle_args(command) if echo: log.info('$ ' + (' '.join(command))) try: return subprocess.Popen(command, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() except Exception as e: log.error(repr(command)) if critical: raise e log.error(e) return False
def _resolveTagNoChdir(self, tag, quiet=None): if quiet is None: quiet = self.quiet ret = cmd_output( ['git', 'rev-list', '-n', '1', 'refs/tags/{}'.format(tag)], echo=not quiet, env=self.noPasswordEnv) if not ret: return None stdout, stderr = ret for line in (stdout + stderr).decode('utf-8').split('\n'): line = line.strip() if line == '': continue if line.startswith('fatal:'): log.error('[git] ' + line) return None return line.strip() return None
def Load(self, filename, merge=False, defaults=None, variables={}): with log.info("Loading %s...", filename): if not os.path.isfile(filename): if defaults is None: log.error('Failed to load %s.', filename) return False else: log.warn('File not found, loading defaults.') with open(filename, 'w') as f: yaml.dump(defaults, f, default_flow_style=False) template = self.environment.get_template(filename) rendered = template.render(variables) newcfg = yaml.load(rendered) if merge: self.cfg = dict_merge(self.cfg, newcfg) else: self.cfg = newcfg return True
def GetRemoteState(self, remote='origin', branch='master'): with Chdir(self.path, quiet=self.quiet): ret = cmd_output(['git', 'fetch', '-q'], echo=not self.quiet) if not ret: return False stdout, stderr = ret for line in (stdout + stderr).split('\n'): line = line.strip() if line == '': continue if line.startswith('fatal:'): log.error('[git] ' + line) return False remoteinfo = Git.LSRemote(remote, branch) if remoteinfo is None: return False ref = 'refs/heads/' + branch if ref in remoteinfo: self.remote_commit = remoteinfo[ref] return True
def UpdateRemotes(self, remote=None, quiet=None): if quiet is None: quiet = self.quiet if remote is not None: self.remotes[remote] = self._syncRemote(remote, quiet=quiet) return True stdout, stderr = cmd_output(['git', 'remote', 'show'], echo=not quiet, env=self.noPasswordEnv) for oline in (stdout + stderr).decode('utf-8').split('\n'): line = oline.strip() if not quiet or not quiet: print(oline) if line == '': continue if line.startswith('fatal:'): log.error('[git] ' + line) return False self.remotes[line] = self._syncRemote(line, quiet=quiet) return True
def cmd(command, echo=False, env=None, show_output=True, critical=False, globbify=True, acceptable_exit_codes=[0]): new_env = _cmd_handle_env(env) command = _cmd_handle_args(command, globbify) if echo: log.info('$ ' + _args2str(command)) output = '' try: if show_output: code = subprocess.call(command, env=new_env, shell=False) #print(repr(code)) success = code in acceptable_exit_codes if critical and not success: raise CalledProcessError(code, command) return success else: # Using our own customized check_output for acceptable_exit_codes. output = check_output(command, env=new_env, stderr=subprocess.STDOUT, acceptable_exit_codes=acceptable_exit_codes) return True except CalledProcessError as cpe: log.error(cpe.output) if critical: raise cpe log.error(cpe) return False except Exception as e: log.error(e) log.error(output) if critical: raise e log.error(e) return False
def cmd_daemonize(command, echo=False, env=None, critical=False): new_env = _cmd_handle_env(env) command = _cmd_handle_args(command) if echo: log.info('& ' + ' '.join(command)) try: if platform.system() == 'Windows': # HACK batch = os.tmpnam() + '.bat' with open(batch, 'w') as b: b.write(' '.join(command)) os.startfile(batch) else: subprocess.Popen(command, env=new_env) return True except Exception as e: log.error(repr(command)) if critical: raise e log.error(e) return False
def DpkgSearchFiles(files): '''Find packages for a given set of files.''' stdout, stderr = cmd_output(['dpkg', '--search'] + files, critical=True) ''' libc6:amd64: /lib/x86_64-linux-gnu/libc-2.19.so libcap2:amd64: /lib/x86_64-linux-gnu/libcap.so.2 libcap2:amd64: /lib/x86_64-linux-gnu/libcap.so.2.24 libc6:amd64: /lib/x86_64-linux-gnu/libcidn-2.19.so libc6:amd64: /lib/x86_64-linux-gnu/libcidn.so.1 libcomerr2:amd64: /lib/x86_64-linux-gnu/libcom_err.so.2 libcomerr2:amd64: /lib/x86_64-linux-gnu/libcom_err.so.2.1 libc6:amd64: /lib/x86_64-linux-gnu/libcrypt-2.19.so libcryptsetup4:amd64: /lib/x86_64-linux-gnu/libcryptsetup.so.4 libcryptsetup4:amd64: /lib/x86_64-linux-gnu/libcryptsetup.so.4.6.0 libc6:amd64: /lib/x86_64-linux-gnu/libcrypt.so.1 libc6:amd64: /lib/x86_64-linux-gnu/libc.so.6 ''' packages = [] if stdout or stderr: for line in (stdout + stderr).decode('utf-8').split('\n'): line = line.strip() if line == '': continue chunks = line.split() # libc6:amd64: /lib/x86_64-linux-gnu/libc.so.6 if len(chunks) == 2: pkgName = chunks[0][:-1] # Strip ending colon if pkgName not in packages: packages += [pkgName] else: log.error('UNHANDLED dpkg --search LINE (len == %d): "%s"', len(chunks), line) return packages
def DpkgSearchFiles(files): '''Find packages for a given set of files.''' stdout, stderr = cmd_output(['dpkg', '--search'] + files, critical=True) ''' libc6:amd64: /lib/x86_64-linux-gnu/libc-2.19.so libcap2:amd64: /lib/x86_64-linux-gnu/libcap.so.2 libcap2:amd64: /lib/x86_64-linux-gnu/libcap.so.2.24 libc6:amd64: /lib/x86_64-linux-gnu/libcidn-2.19.so libc6:amd64: /lib/x86_64-linux-gnu/libcidn.so.1 libcomerr2:amd64: /lib/x86_64-linux-gnu/libcom_err.so.2 libcomerr2:amd64: /lib/x86_64-linux-gnu/libcom_err.so.2.1 libc6:amd64: /lib/x86_64-linux-gnu/libcrypt-2.19.so libcryptsetup4:amd64: /lib/x86_64-linux-gnu/libcryptsetup.so.4 libcryptsetup4:amd64: /lib/x86_64-linux-gnu/libcryptsetup.so.4.6.0 libc6:amd64: /lib/x86_64-linux-gnu/libcrypt.so.1 libc6:amd64: /lib/x86_64-linux-gnu/libc.so.6 ''' packages = [] if stdout or stderr: for line in (stdout + stderr).split('\n'): line = line.strip() if line == '': continue chunks = line.split() # libc6:amd64: /lib/x86_64-linux-gnu/libc.so.6 if len(chunks) == 2: pkgName = chunks[0][:-1] # Strip ending colon if pkgName not in packages: packages += [pkgName] else: log.error('UNHANDLED dpkg --search LINE (len == %d): "%s"', len(chunks), line) return packages
def GetRemoteState(self, remote='origin', branch='HEAD', quiet=None): if quiet is None: quiet = self.quiet with Chdir(self.path, quiet=self.quiet): ret = cmd_output( ['git', 'fetch', '-q', '--all', '--prune', '--tags'], echo=not quiet, env=self.noPasswordEnv) if not ret: return False stdout, stderr = ret for line in (stdout + stderr).decode('utf-8').split('\n'): # if not quiet: # print(line) line = line.strip() if line == '': continue if line.startswith('fatal:'): log.error('[git] ' + line) return False for _remote in self.remotes.values(): _remote.fetched = False self.UpdateRemotes(remote=remote, quiet=quiet) if branch == 'HEAD': branch = self.remotes[remote].findBranch(branch) remoteinfo = Git.LSRemote(remote, branch, quiet=quiet) if remoteinfo is None: return False if branch == 'HEAD': ref = 'HEAD' elif '/' not in branch: ref = 'refs/heads/' + branch if ref in remoteinfo: self.remote_commit = remoteinfo[ref] return True
def cmd_output(command, echo=False, env=None, critical=False, globbify=True) -> Tuple[bytes, bytes]: ''' :returns List[2]: (stdout,stderr) ''' new_env = _cmd_handle_env(env) command = _cmd_handle_args(command, globbify) if echo: log.info('$ ' + _args2str(command)) try: return subprocess.Popen(command, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() except Exception as e: log.error(repr(command)) if critical: raise e log.error(e) return False