def build(self): os_utils.ensureDirExists(os.path.dirname(self.target)) stdout, stderr = os_utils.cmd_output([self.js2coffee_path]+self.files+self.js2coffee_opts, echo=self.should_echo_commands(), critical=True) if stderr.strip() != '': log.error(stderr) with codecs.open(self.target, 'w', encoding='utf-8-sig') as outf: outf.write(stdout)
def LSRemote(cls, remote=None, ref=None, quiet=True): args = [] if remote: args.append(remote) if ref: args.append(ref) try: ret = cmd_output(['git', 'ls-remote'] + args, echo=not quiet, critical=True) if not ret: return None #print(repr(ret)) stderr, stdout = ret o = {} for line in (stdout + stderr).decode('utf-8').split('\n'): line = line.strip() if not quiet: print(line) if line == '': continue line_chunks = line.split() if len(line_chunks) == 2: hashid, ref = line_chunks o[ref] = hashid return o except Exception as e: print(e) pass return None
def _hgcmd(self, args): stdout, stderr = cmd_output(['hg', '--cwd', self.path, '--encoding', 'UTF-8'] + args, echo=not self.quiet, critical=True) if self.show_output: with log: for line in (stdout + stderr).decode('utf-8').split('\n'): if line.strip() == '': continue log.info('-> %s', line) return (stdout + stderr)
def _hgcmd(self, args): stdout, stderr = cmd_output(['hg', '--cwd', self.path, '--encoding', 'UTF-8'] + args, echo=not self.quiet, critical=True) if self.show_output: with log: for line in (stdout + stderr).split('\n'): if line.strip() == '': continue log.info('-> %s', line) return (stdout + stderr)
def checkHg(): '''Will raise CalledProcessError if something goes sideways.''' global HG_VERSION if HG_VERSION is None: stdout, stderr = cmd_output(['hg', '--version'], critical=True) for line in (stdout + stderr).split('\n'): m = REG_VERSION.search(line) if m: HG_VERSION = m.group(1) break log.info('mercurial version %s detected.', HG_VERSION)
def checkHg(): '''Will raise CalledProcessError if something goes sideways.''' global HG_VERSION if HG_VERSION is None: stdout, stderr = cmd_output(['hg', '--version'], critical=True) for line in (stdout + stderr).decode('utf-8').split('\n'): m = REG_VERSION.search(line) if m: HG_VERSION = m.group(1) break log.info('mercurial version %s detected.', HG_VERSION)
def UpdateRemotes(self): stdout, stderr = cmd_output(['git', 'remote', 'show'], echo=not self.quiet) for line in (stdout + stderr).split('\n'): line = line.strip() if line == '': continue if line.startswith('fatal:'): log.error('[git] ' + line) return False self.remotes[line] = self._getRemoteInfo(line) return True
def GetBranch(cls, quiet=True): try: stderr, stdout = cmd_output( ["git", "rev-parse", "--abbrev-ref", 'HEAD'], echo=not quiet, critical=True) return (stderr + stdout).strip() # branch = subprocess.Popen(["git", "rev-parse", "--abbrev-ref", 'HEAD'], stdout=subprocess.PIPE).communicate()[0][:-1] # if branch: # return branch.decode('utf-8') except Exception as e: print(e) pass return '[UNKNOWN]'
def GetBranch(cls, quiet=True): try: cmd = ["git", "symbolic-ref", "--short", 'HEAD'] #stderr, stdout = cmd_output(["git", "rev-parse", "--abbrev-ref", 'HEAD', '--'], echo=not quiet, critical=True) stderr, stdout = cmd_output(cmd, echo=not quiet, critical=True) o = (stderr + stdout).decode('utf-8').strip() if cls.OutputIsFatal(o): log.error(_args2str(cmd)) return None return o.strip() except Exception as e: print(e) pass return None
def IsDirty(cls, quiet=True): try: # branch = subprocess.Popen(['git', 'ls-files', '-m', '-o', '-d', '--exclude-standard'], stdout=subprocess.PIPE).communicate()[0][:-1] # if branch: stderr, stdout = cmd_output( ['git', 'ls-files', '-m', '-o', '-d', '--exclude-standard'], echo=not quiet, critical=True) for line in (stderr + stdout).decode('utf-8').split('\n'): line = line.strip() if line != '': return True return False except Exception as e: print(e) pass return None
def GetCommit(cls, ref='HEAD', short=True, quiet=True): try: addtl_flags = [] if short: addtl_flags.append('--short') stdout, stderr = cmd_output(['git', 'rev-parse'] + addtl_flags + [ref], echo=not quiet, critical=True) return (stdout + stderr).decode('utf-8').strip() # rev = subprocess.Popen(['git', 'rev-parse'] + addtl_flags + [ref], stdout=subprocess.PIPE).communicate()[0][:-1] # if rev: # return rev.decode('utf-8') except Exception as e: print(e) pass return '[UNKNOWN]'
def _resolveTagNoChdir(self, tag, quiet=None): if quiet is None: quiet = self.quiet ret = cmd_output( ['git', 'rev-list', '-n', '1', 'refs/tags/{}'.format(tag)], echo=not quiet, env=self.noPasswordEnv) if not ret: return None stdout, stderr = ret for line in (stdout + stderr).decode('utf-8').split('\n'): line = line.strip() if line == '': continue if line.startswith('fatal:'): log.error('[git] ' + line) return None return line.strip() return None
def GetRemoteState(self, remote='origin', branch='master'): with Chdir(self.path, quiet=self.quiet): ret = cmd_output(['git', 'fetch', '-q'], echo=not self.quiet) if not ret: return False stdout, stderr = ret for line in (stdout + stderr).split('\n'): line = line.strip() if line == '': continue if line.startswith('fatal:'): log.error('[git] ' + line) return False remoteinfo = Git.LSRemote(remote, branch) if remoteinfo is None: return False ref = 'refs/heads/' + branch if ref in remoteinfo: self.remote_commit = remoteinfo[ref] return True
def UpdateRemotes(self, remote=None, quiet=None): if quiet is None: quiet = self.quiet if remote is not None: self.remotes[remote] = self._syncRemote(remote, quiet=quiet) return True stdout, stderr = cmd_output(['git', 'remote', 'show'], echo=not quiet, env=self.noPasswordEnv) for oline in (stdout + stderr).decode('utf-8').split('\n'): line = oline.strip() if not quiet or not quiet: print(oline) if line == '': continue if line.startswith('fatal:'): log.error('[git] ' + line) return False self.remotes[line] = self._syncRemote(line, quiet=quiet) return True
def _getRemoteInfo(self, remoteID): ''' $ git remote show origin * remote origin Fetch URL: https://github.com/d3athrow/vgstation13.git Push URL: https://github.com/d3athrow/vgstation13.git HEAD branch: Bleeding-Edge Remote branches: Bleeding-Edge tracked returns: https://github.com/d3athrow/vgstation13.git ''' stdout, stderr = cmd_output(['git', 'remote', 'show', remoteID], echo=not self.quiet) for line in (stdout + stderr).split('\n'): line = line.strip() components = line.split() if line.startswith('Fetch URL:'): # self.remotes[remoteID]=line[2] return line[2]
def _syncRemote(self, remoteID, quiet=None): ''' $ git remote show origin * remote origin Fetch URL: https://github.com/d3athrow/vgstation13.git Push URL: https://github.com/d3athrow/vgstation13.git HEAD branch: Bleeding-Edge Remote branches: Bleeding-Edge tracked returns: GitRemoteInfo() ''' o = GitRemoteInfo() o.id = remoteID if quiet is None: quiet = self.quiet stdout, stderr = cmd_output(['git', 'remote', 'show', remoteID], echo=not quiet, env=self.noPasswordEnv) in_branches = False for oline in (stdout + stderr).decode('utf-8').split('\n'): line = oline.strip() if not quiet: print(oline) components = line.split() if not in_branches: if line.startswith('Fetch URL:'): o.fetch_uri = components[2] if line.startswith('Push URL:'): o.push_uri = components[2] if line.startswith('HEAD branch:'): o.head_branch = components[2] if line.startswith('Remote branches:'): in_branches = True else: o.branches.append(line.split(' ')[0]) o.fetched = True return o
def GetRemoteState(self, remote='origin', branch='HEAD', quiet=None): if quiet is None: quiet = self.quiet with Chdir(self.path, quiet=self.quiet): ret = cmd_output( ['git', 'fetch', '-q', '--all', '--prune', '--tags'], echo=not quiet, env=self.noPasswordEnv) if not ret: return False stdout, stderr = ret for line in (stdout + stderr).decode('utf-8').split('\n'): # if not quiet: # print(line) line = line.strip() if line == '': continue if line.startswith('fatal:'): log.error('[git] ' + line) return False for _remote in self.remotes.values(): _remote.fetched = False self.UpdateRemotes(remote=remote, quiet=quiet) if branch == 'HEAD': branch = self.remotes[remote].findBranch(branch) remoteinfo = Git.LSRemote(remote, branch, quiet=quiet) if remoteinfo is None: return False if branch == 'HEAD': ref = 'HEAD' elif '/' not in branch: ref = 'refs/heads/' + branch if ref in remoteinfo: self.remote_commit = remoteinfo[ref] return True
def _git(self, args, echo=False): return cmd_output(['git'] + args, echo=echo, env=self.noPasswordEnv)
def build(self): gitmodules = {} with open(self.gitmodulesfile, 'r') as tomlf: smid = None for line in tomlf: line = line.strip() m = REG_SUBMODULE_SECTION.match(line) if m is not None: smid = m.group(1).strip() gitmodules[smid] = {} if '=' in line: k, v = line.split('=', 2) gitmodules[smid][k.strip()] = v.strip() gitconfig = {} with open(self.gitconfigfile, 'r') as tomlf: smid = None for line in tomlf: line = line.strip() #print(line) m = REG_SUBMODULE_SECTION.match(line) if m is not None: smid = m.group(1).strip() gitconfig[smid] = {} if smid is not None and '=' in line: #print(line) k, v = line.split('=', 2) gitconfig[smid][k.strip()] = v.strip() ''' with open(self.gitmodulesfile + '.yml', 'w') as f: yaml.dump(gitmodules, f, default_flow_style=False) with open('.gitconfig.yml', 'w') as f: yaml.dump(gitconfig, f, default_flow_style=False) ''' for repoID, repoconf in gitconfig.items(): if repoID not in gitmodules.keys(): with log.warn('Submodule %s is present in .git/config but not .gitmodules!', repoID): pathspec = repoconf.get('path', repoID) path = os.path.abspath(pathspec) tag = repoconf.get('tag', None) branch = repoconf.get('branch', 'HEAD' if tag is None else None) log.info('path = %s', pathspec) for repoID, repoconf in gitmodules.items(): if repoID not in gitconfig.keys(): with log.warn('Submodule %s is present in .gitmodules but not .git/config!', repoID): pathspec = repoconf.get('path', repoID) path = os.path.abspath(pathspec) tag = repoconf.get('tag', None) branch = repoconf.get('branch', 'HEAD' if tag is None else None) opts = [] if branch != 'HEAD': opts += ['-b', branch] log.info('path = %s', pathspec) if os.path.isdir(path): log.warn('Removing existing %s directory.', path) shutil.rmtree(path) cmd = ['git', 'submodule', 'add']+opts+['-f', '--name', repoID, '--', repoconf.get('url'), pathspec] os_utils.cmd(cmd, critical=True, echo=self.should_echo_commands(), show_output=True) #log.error('Would exec: %s', ' '.join(cmd)) for repoID, repoconf in gitmodules.items(): with log.info('Checking %s...', repoID): pathspec = repoconf.get('path', repoID) path = os.path.abspath(pathspec) tag = repoconf.get('tag', None) branch = repoconf.get('branch', 'HEAD' if tag is None else None) if os.path.isdir(path): desired_commit = '' cmdline = ['git', 'ls-tree', Git.GetBranch(), pathspec] stdout, stderr = os_utils.cmd_output(cmdline, echo=self.should_echo_commands(), critical=True) skip_this = False for line in (stdout+stderr).decode('utf-8').splitlines(): if line.startswith('error:') or line.startswith('fatal:'): log.critical(line) raise error.SubprocessThrewError(cmdline, line) line,repoID = line.strip().split('\t') _, _, desired_commit = line.split(' ') if not skip_this: with os_utils.Chdir(path, quiet=not self.should_echo_commands()): cur_commit = Git.GetCommit(short=False, quiet=not self.should_echo_commands()) #log.info(desired_commit) #log.info(cur_commit) if cur_commit == desired_commit: log.info('Commits are synced, skipping.') continue repo = GitRepository(path, origin_uri=repoconf['url'], submodule=True) if repo.CheckForUpdates(branch=branch, quiet=False): if os.path.isdir(path): os_utils.cmd(['git', 'submodule', 'sync', '--', pathspec], critical=True, echo=self.should_echo_commands(), show_output=True) os_utils.cmd(['git', 'submodule', 'update', '--init', '--recursive', pathspec], critical=True, echo=self.should_echo_commands(), show_output=True)
def _git(self, args, echo=False): ret = cmd_output(['git'] + args, echo=echo)