def set(self, key, val, noisy=None): if noisy is None: noisy = self.noisy key = self.getKey(key) if noisy: log.info('Environment: {} = {}'.format(key, val)) self.env[key] = val
def cmd(command, echo=False, env=None, show_output=True, critical=False): new_env = _cmd_handle_env(env) command = _cmd_handle_args(command) if echo: log.info('$ ' + (' '.join(command))) if show_output: return subprocess.call(command, env=new_env, shell=False) == 0 output = '' try: output = subprocess.check_output(command, env=new_env, stderr=subprocess.STDOUT) return True except CalledProcessError as cpe: log.error(cpe.output) if critical: raise cpe log.error(cpe) return False except Exception as e: log.error(e) log.error(output) if critical: raise e log.error(e) return False
def del_empty_dirs(src_dir: str, quiet=False) -> int: ''' Removes empty directories. :param src_dir: Root of directory tree to search for empty directories. :param quiet: Squelches log messages about removing empty directories. :returns: Count of removed directories. ''' ndeleted = -1 totalDel = 0 while ndeleted != 0: ndeleted = 0 # Listing the files for dirpath, dirnames, filenames in os.walk(src_dir, topdown=False): #print(dirpath, src_dir) if dirpath == src_dir: continue #print(dirpath, len(dirnames), len(filenames)) if len(filenames) == 0 and len(dirnames) == 0: if not quiet: log.info('Removing %s (empty)...', dirpath) os.rmdir(dirpath) ndeleted += 1 totalDel += 1 return totalDel
def cmd_daemonize(command, echo=False, env=None, critical=False, globbify=True): new_env = _cmd_handle_env(env) command = _cmd_handle_args(command, globbify) if echo: log.info('& ' + _args2str(command)) try: if platform.system() == 'Windows': # HACK batch = os.tmpnam() + '.bat' with open(batch, 'w') as b: b.write(' '.join(command)) os.startfile(batch) else: subprocess.Popen(command, env=new_env) return True except Exception as e: log.error(repr(command)) if critical: raise e log.error(e) return False
def cmd_out(command: Union[List[str], str], echo=False, env=None, critical=False, globbify=True, encoding: str = 'utf-8') -> str: ''' :returns str: stderr and stdout, piped into one string and decoded as UTF-8. ''' new_env = _cmd_handle_env(env) command = _cmd_handle_args(command, globbify) if echo: log.info('$ ' + _args2str(command)) try: p = subprocess.Popen(command, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) return p.stdout.read().decode('utf-8') except Exception as e: log.error(repr(command)) if critical: raise e log.error(e) return None
def _dump_to_callback(self, bid, cb): buf = self.buf[bid].strip() if self.debug: log.info('%s buf = %r', self._logPrefix(), buf) if buf != '': cb(self._asyncCommand, buf) self.buf[bid] = ''
def Load(self, filename, default=None, expand_vars={}): with log.info("Loading %s...", filename): if not os.path.isfile(filename): if default is None: log.critical('File not found, exiting!') log.info( 'To load defaults, specify default=<dict> to Properties.Load().' ) sys.exit(1) else: log.warn('File not found, loading defaults.') with open(filename, 'w') as f: self.dumpTo(default, f) with open(filename, 'r') as f: for line in f: if line.strip() == '': continue # Ignore comments. if line.strip().startswith('#'): continue key, value = line.strip().split('=', 1) if key in self.properties: log.warn('Key "%s" already exists, overwriting!', key) value = replace_vars(value, expand_vars) self.properties[key] = value if default is None: return for k, v in default.items(): if k not in self.properties: self.properties[k] = v log.info('Added default property %s = %s', k, v)
def _processData(self, bid: str, cb, data: bytes): if self.debug: log.info('%s %s: Received %d bytes', self._logPrefix(), bid, len(data)) for b in bytes2str(data): if b not in self.linebreaks: self.buf[bid] += b else: self._dump_to_callback(bid, cb)
def appendTo(self, key, value, delim=';', noisy=None): if noisy is None: noisy = self.noisy key = self.getKey(key) if noisy: log.info('Environment: {1} appended to {0}'.format(key, value)) self.env[key] = delim.join( self.env.get(key, '').split(delim) + [value])
def _hgcmd(self, args): stdout, stderr = cmd_output(['hg', '--cwd', self.path, '--encoding', 'UTF-8'] + args, echo=not self.quiet, critical=True) if self.show_output: with log: for line in (stdout + stderr).split('\n'): if line.strip() == '': continue log.info('-> %s', line) return (stdout + stderr)
def _hgcmd(self, args): stdout, stderr = cmd_output(['hg', '--cwd', self.path, '--encoding', 'UTF-8'] + args, echo=not self.quiet, critical=True) if self.show_output: with log: for line in (stdout + stderr).decode('utf-8').split('\n'): if line.strip() == '': continue log.info('-> %s', line) return (stdout + stderr)
def assertWhich(self, program, fail_raise=False, skip_paths=[]): fullpath = self.which(program, skip_paths) with log.info('Checking if %s exists...', program): if fullpath is None: errmsg = '{executable} is not in PATH!'.format( executable=program) raise RuntimeError(errmsg) else: log.info('Found: %s', fullpath) return fullpath
def __enter__(self): try: if os.getcwdu() != self.chdir: os.chdir(self.chdir) if not self.quiet: log.info('cd ' + self.chdir) except: log.critical('Failed to chdir to {}.'.format(self.chdir)) sys.exit(1) return self
def __exit__(self, typeName, value, traceback): try: if os.getcwdu() != self.pwd: os.chdir(self.pwd) if not self.quiet: log.info('cd ' + self.pwd) except: log.critical('Failed to chdir to {}.'.format(self.pwd)) sys.exit(1) return False
def AddAssemblyRef(self, refID, hintpath, verbose=False): if verbose: log.info('Adding assembly reference %s.', refID) reference = self.subelement(self._referenceGroup, 'Reference') asmRef = AssemblyReference(self, reference) asmRef.Include = refID asmRef.HintPath = hintpath self.ReferencesByName[asmRef.RefID] = asmRef self.References.append(asmRef) return asmRef
def prependTo(self, key, value, delim=None, noisy=None): if delim is None: delim = os.pathsep if noisy is None: noisy = self.noisy key = self.getKey(key) if noisy: log.info('Environment: {1} prepended to {0}'.format(key, value)) self.env[key] = delim.join([value] + self.env.get(key, '').split(delim))
def configure_cotire(cfg, cmake): global ENV with log.info('Configuring cotire...'): if not cfg.get('env.cotire.enabled', False): log.info('cotire disabled, skipping.') else: ENV.set('CCACHE_SLOPPINESS', 'time_macros') cmake.setFlag('ENABLE_COTIRE', 'On') if cfg.get('env.make.jobs', 1) > 1: cmake.setFlag('COTIRE_MAXIMUM_NUMBER_OF_UNITY_INCLUDES', cfg.get('env.make.jobs', 1))
def checkHg(): '''Will raise CalledProcessError if something goes sideways.''' global HG_VERSION if HG_VERSION is None: stdout, stderr = cmd_output(['hg', '--version'], critical=True) for line in (stdout + stderr).decode('utf-8').split('\n'): m = REG_VERSION.search(line) if m: HG_VERSION = m.group(1) break log.info('mercurial version %s detected.', HG_VERSION)
def checkHg(): '''Will raise CalledProcessError if something goes sideways.''' global HG_VERSION if HG_VERSION is None: stdout, stderr = cmd_output(['hg', '--version'], critical=True) for line in (stdout + stderr).split('\n'): m = REG_VERSION.search(line) if m: HG_VERSION = m.group(1) break log.info('mercurial version %s detected.', HG_VERSION)
def AddProjectRef(self, include, name, guid, verbose=False): if verbose: log.info('Adding project reference %s.', name) reference = self.subelement(self._projectReferenceGroup, "ProjectReference") projRef = ProjectReference(self, reference) self.ReferencesByName[projRef.RefID] = projRef self.References.append(projRef) projRef.Name = name projRef.Include = include projRef.Project = guid return projRef
def _processData(self, bid, cb, data): if self.debug: log.info('%s %s: Received %d bytes', self._logPrefix(), bid, len(data)) for b in data: if b != '\n' and b != '\r' and b != '': self.buf[bid] += b else: buf = self.buf[bid].strip() if self.debug: log.info('buf = %r', buf) if buf != '': cb(self._asyncCommand, buf) self.buf[bid] = ''
def removeDuplicatedEntries(self, key, noisy=None, delim=os.pathsep): if noisy is None: noisy = self.noisy newlist = [] key = self.getKey(key) for entry in self.env[key].split(delim): entry = entry.strip('"') if entry in newlist: if noisy: log.info( 'Build env: Removing %r from %s: duplicated entry.', entry, key) continue newlist += [entry] self.env[key] = delim.join(newlist)
def Pull(self, remote='origin', branch='HEAD', commit=None, tag=None, cleanup=False): if branch == 'HEAD': branch = self.remotes[remote].head_branch if self.submodule: log.error('Submodules should not call Pull!') return if not os.path.isdir(self.path): cmd(['git', 'clone', self.remotes[remote].fetch_uri, self.path], echo=not self.quiet or self.noisy_clone, critical=True, show_output=not self.quiet or self.noisy_clone, env=self.noPasswordEnv) with Chdir(self.path, quiet=self.quiet): if cleanup: cmd(['git', 'clean', '-fdx'], echo=not self.quiet, critical=True) cmd(['git', 'reset', '--hard'], echo=not self.quiet, critical=True) if self.current_branch != branch: ref = 'remotes/{}/{}'.format(remote, branch) cmd(['git', 'checkout', '-B', branch, ref, '--'], echo=not self.quiet, critical=True) if tag is not None: commit = self._resolveTagNoChdir(tag) if commit is not None: cmd(['git', 'checkout', commit], echo=not self.quiet, critical=True) else: if self.current_commit != self.remote_commit: cmd([ 'git', 'reset', '--hard', '{}/{}'.format( remote, branch) ], echo=not self.quiet, critical=True) if self.UsesLFS(): log.info('git-lfs detected!') cmd(['git', 'lfs', 'pull'], echo=not self.quiet, critical=True) return True
def __init__(self, scope): log.info('Python version: 0x%0.8X' % sys.hexversion) if sys.hexversion > 0x03000000: import winreg #IGNORE:import-error else: import _winreg as winreg #IGNORE:import-error self.winreg = winreg assert scope in ('user', 'system') self.scope = scope if scope == 'user': self.root = winreg.HKEY_CURRENT_USER self.subkey = 'Environment' else: self.root = winreg.HKEY_LOCAL_MACHINE self.subkey = r'SYSTEM\CurrentControlSet\Control\Session Manager\Environment'
def cmd_output(command, echo=False, env=None, critical=False): ''' :returns List[2]: (stdout,stderr) ''' new_env = _cmd_handle_env(env) command = _cmd_handle_args(command) if echo: log.info('$ ' + (' '.join(command))) try: return subprocess.Popen(command, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() except Exception as e: log.error(repr(command)) if critical: raise e log.error(e) return False
def logStart(self): color = self.BT_COLOR or 'cyan' pct = round((len(self.maestro.targetsCompleted)/len(self.maestro.targets))*100) msg = '' if self.maestro.verbose: msg = self.verboseLogEntry(color) else: msg = self.standardLogEntry(color) return log.info(f'[{str(pct).rjust(3)}%] {msg}')
def LoadDebianDirectory(self, dir): '''Loads DEBIAN/ directory.''' file_conf = os.path.join(dir, 'conffiles') if os.path.isfile(file_conf): newconfigs = self._file2list(file_conf) if len(newconfigs) > 0: self.configs += newconfigs log.info('Added %r to configs', ', '.join(newconfigs)) def setIfIsFile(prop, filename): if os.path.isfile(filename): setattr(self, prop, filename) log.info('Set %r to %r', prop, filename) setIfIsFile('afterInstall', os.path.join(dir, 'postinst')) setIfIsFile('beforeInstall', os.path.join(dir, 'preinst')) setIfIsFile('afterRemove', os.path.join(dir, 'postrm')) setIfIsFile('beforeRemove', os.path.join(dir, 'prerm'))
def InstallDpkgPackages(packages): import apt # IGNORE:import-error with log.info('Checking dpkg packages...'): cache = apt.Cache() num_changes = 0 with cache.actiongroup(): for pkg in packages: if pkg not in cache: log.critical('UNKNOWN APT PACKAGE {}!'.format(pkg)) sys.exit(1) package = cache[pkg] if not package.is_installed: package.mark_install() num_changes += 1 if num_changes == 0: log.info('No changes required, skipping.') return cache.commit(apt.progress.text.AcquireProgress(), apt.progress.base.InstallProgress())
def single_copy(fromfile: str, newroot: str, **op_args): ''' :param as_file bool: Copy to new name rather than to new directory. False by default. :param verbose bool: Log copying action. :param ignore_mtime bool: Ignore file modification timestamps. :param ignore_filecmp bool: Disable byte-to-byte comparison AND os.stat checks. :param ignore_bytecmp bool: Do not check each file byte-for-byte, perform shallow os.stat checks. ''' newfile = os.path.join(newroot, os.path.basename(fromfile)) if op_args.get('as_file', False) or '.' in newroot: newfile = newroot if canCopy(fromfile, newfile, **op_args): if op_args.get('verbose', False): log.info('Copying {} -> {}'.format(fromfile, newfile)) shutil.copy2(fromfile, newfile)
def cmd(command, echo=False, env=None, show_output=True, critical=False, globbify=True, acceptable_exit_codes=[0]): new_env = _cmd_handle_env(env) command = _cmd_handle_args(command, globbify) if echo: log.info('$ ' + _args2str(command)) output = '' try: if show_output: code = subprocess.call(command, env=new_env, shell=False) #print(repr(code)) success = code in acceptable_exit_codes if critical and not success: raise CalledProcessError(code, command) return success else: # Using our own customized check_output for acceptable_exit_codes. output = check_output(command, env=new_env, stderr=subprocess.STDOUT, acceptable_exit_codes=acceptable_exit_codes) return True except CalledProcessError as cpe: log.error(cpe.output) if critical: raise cpe log.error(cpe) return False except Exception as e: log.error(e) log.error(output) if critical: raise e log.error(e) return False
def build(self, CMAKE, dir='.', env=None, target=None, moreflags=[]): if env is None: env = ENV.env flags = ['--build', dir] if target is not None: moreflags += ['--target', target] flags += moreflags with log.info('Running CMake --build:'): BuildEnv.dump(env) return cmd([CMAKE] + flags, env=env, critical=True, echo=True)
def UpdateSubmodules(self, remote=False): with log.info('Updating submodules in %s...', self.path): with Chdir(self.path, quiet=self.quiet): if os.path.isfile('.gitmodules'): more_flags = [] if remote: more_flags.append('--remote') cmd([ 'git', 'submodule', 'update', '--init', '--recursive' ] + more_flags, echo=not self.quiet, critical=True)
def optree(fromdir, todir, op, ignore=None, **op_args): if ignore is None: ignore = [] # print('ignore=' + repr(ignore)) for root, _, files in os.walk(fromdir): path = root.split(os.sep) start = len(fromdir) if root[start:].startswith(os.sep): start += 1 substructure = root[start:] assert not substructure.startswith(os.sep) newroot = os.path.join(todir, substructure) if any([(x + '/' in ignore) for x in path]): if op_args.get('verbose', False): log.info(u'Skipping {}'.format(substructure)) continue if not os.path.isdir(newroot): if op_args.get('verbose', False): log.info(u'mkdir {}'.format(newroot)) os.makedirs(newroot) for filename in files: fromfile = os.path.join(root, filename) _, ext = os.path.splitext(os.path.basename(fromfile)) if ext in ignore: if op_args.get('verbose', False): log.info(u'Skipping {} ({})'.format(fromfile, ext)) continue op(fromfile, newroot, **op_args)
def cmd_daemonize(command, echo=False, env=None, critical=False): new_env = _cmd_handle_env(env) command = _cmd_handle_args(command) if echo: log.info('& ' + ' '.join(command)) try: if platform.system() == 'Windows': # HACK batch = os.tmpnam() + '.bat' with open(batch, 'w') as b: b.write(' '.join(command)) os.startfile(batch) else: subprocess.Popen(command, env=new_env) return True except Exception as e: log.error(repr(command)) if critical: raise e log.error(e) return False
def configure_ccache(cfg, cmake): global ENV with log.info('Configuring ccache...'): if not cfg.get('env.ccache.enabled', False): log.info('ccache disabled, skipping.') # Otherwise, strange things happen. ENV.set('CC', _which_if_basename(ENV.get('CC','gcc') + '.real')) ENV.set('CXX', _which_if_basename(ENV.get('CXX','g++') + '.real')) else: CCACHE = _which_if_basename(cfg.get('bin.ccache', 'ccache')) DISTCC = _which_if_basename(cfg.get('bin.distcc', 'distcc')) if cfg.get('env.distcc.enabled', False): ENV.set('CCACHE_PREFIX', DISTCC) # Fixes a bug where CMake sets this all incorrectly. # http://public.kitware.com/Bug/view.php?id=12274 cmake.setFlag('CMAKE_CXX_COMPILER_ARG1', ENV.env['CXX']) # set_cmake_env('CMAKE_ASM_COMPILER_ARG1',ENV.env['ASM']) ENV.set('CC', CCACHE + ' ' + ENV.env['CC']) ENV.set('CXX', CCACHE + ' ' + ENV.env['CXX']) # ENV.set('ASM',CCACHE + ' ' + ENV.env['ASM'])
def run(self, CMAKE, env=None, dir='.', moreflags=[]): if env is None: env = ENV.env flags = [] if self.generator is not None: flags += ['-G', self.generator] for key, value in self.flags.items(): flags += ['-D{0}={1}'.format(key, value)] flags += moreflags with log.info('Running CMake:'): BuildEnv.dump(env) return cmd([CMAKE] + flags + [dir], env=env, critical=True, echo=True) return False
def findLib(self, givenname): if self.LDPATHS is None: self.LDPATHS = ldpaths() log.info('Search ldpaths: ' + repr(self.LDPATHS)) myclass=self.getElfClass() for path in self.LDPATHS: lib = path + os.sep + givenname if os.path.exists(lib): e = ELFInfo(lib) eclass = e.getElfClass() e.Close() dbg='Found {0}, using {1}'.format(lib, eclass) if myclass == eclass: log.info('%s (== %s)',dbg,myclass) return lib log.info('%s (!= %s), skipped',dbg,myclass) return givenname
def CheckForUpdates(self, remote='default', branch='default', quiet=True): if self.repo is None: return True if not quiet: log.info('Checking %s for updates...', self.path) with log: if not os.path.isdir(self.path): return True self.GetRepoState() self.GetRemoteState(remote, branch) if self.current_branch != branch: if not quiet: log.info('Branch is wrong! %s (L) != %s (R)', self.current_branch, branch) return True if self.current_rev != self.remote_rev: if not quiet: log.info('Revision is out of date! %s (L) != %s (R)', self.current_rev, self.remote_rev) return True return False
def dump(cls, env): for key, value in sorted(env.iteritems()): log.info('+{0}="{1}"'.format(key, value))
def setFlag(self, key, val): log.info('CMake: {} = {}'.format(key, val)) self.flags[key] = val
def Stop(cls): reactor.stop() log.info('Twisted Reactor stopped.')
def __exit__(self, typeName, value, traceback): self.vars['elapsed'] = secondsToStr(clock() - self.start_time) with log: log.info(self.label.toStr(self.vars)) return False
def Start(cls): if cls.instance is None: cls.instance = threading.Thread(target=reactor.run, args=(False,)) cls.instance.daemon = True cls.instance.start() log.info('Twisted Reactor started.')
def ensureDirExists(path, mode=0o777, noisy=False): if not os.path.isdir(path): os.makedirs(path, mode) if noisy: log.info('Created %s.', path)
def single_copy(fromfile, newroot, **op_args): newfile = os.path.join(newroot, os.path.basename(fromfile)) if canCopy(fromfile, newfile, **op_args): if op_args.get('verbose', False): log.info('Copying {} -> {}'.format(fromfile, newfile)) shutil.copy2(fromfile, newfile)
def set(self, key, val): log.info('Build env: {} = {}'.format(key, val)) self.env[key] = val