def build(self):
os_utils.ensureDirExists(os.path.dirname(self.target))
stdout, stderr = os_utils.cmd_output([self.js2coffee_path]+self.files+self.js2coffee_opts, echo=self.should_echo_commands(), critical=True)
if stderr.strip() != '':
log.error(stderr)
with codecs.open(self.target, 'w', encoding='utf-8-sig') as outf:
outf.write(stdout)
def LSRemote(cls, remote=None, ref=None, quiet=True):
args = []
if remote:
args.append(remote)
if ref:
args.append(ref)
try:
ret = cmd_output(['git', 'ls-remote'] + args,
echo=not quiet,
critical=True)
if not ret:
return None
#print(repr(ret))
stderr, stdout = ret
o = {}
for line in (stdout + stderr).decode('utf-8').split('\n'):
line = line.strip()
if not quiet:
print(line)
if line == '':
continue
line_chunks = line.split()
if len(line_chunks) == 2:
hashid, ref = line_chunks
o[ref] = hashid
return o
except Exception as e:
print(e)
pass
return None
def _hgcmd(self, args):
stdout, stderr = cmd_output(['hg', '--cwd', self.path, '--encoding', 'UTF-8'] + args, echo=not self.quiet, critical=True)
if self.show_output:
with log:
for line in (stdout + stderr).decode('utf-8').split('\n'):
if line.strip() == '':
continue
log.info('-> %s', line)
return (stdout + stderr)
def _hgcmd(self, args):
stdout, stderr = cmd_output(['hg', '--cwd', self.path, '--encoding', 'UTF-8'] + args, echo=not self.quiet, critical=True)
if self.show_output:
with log:
for line in (stdout + stderr).split('\n'):
if line.strip() == '':
continue
log.info('-> %s', line)
return (stdout + stderr)
def checkHg():
'''Will raise CalledProcessError if something goes sideways.'''
global HG_VERSION
if HG_VERSION is None:
stdout, stderr = cmd_output(['hg', '--version'], critical=True)
for line in (stdout + stderr).split('\n'):
m = REG_VERSION.search(line)
if m:
HG_VERSION = m.group(1)
break
log.info('mercurial version %s detected.', HG_VERSION)
def checkHg():
'''Will raise CalledProcessError if something goes sideways.'''
global HG_VERSION
if HG_VERSION is None:
stdout, stderr = cmd_output(['hg', '--version'], critical=True)
for line in (stdout + stderr).decode('utf-8').split('\n'):
m = REG_VERSION.search(line)
if m:
HG_VERSION = m.group(1)
break
log.info('mercurial version %s detected.', HG_VERSION)
def UpdateRemotes(self):
stdout, stderr = cmd_output(['git', 'remote', 'show'],
echo=not self.quiet)
for line in (stdout + stderr).split('\n'):
line = line.strip()
if line == '':
continue
if line.startswith('fatal:'):
log.error('[git] ' + line)
return False
self.remotes[line] = self._getRemoteInfo(line)
return True
def GetBranch(cls, quiet=True):
try:
stderr, stdout = cmd_output(
["git", "rev-parse", "--abbrev-ref", 'HEAD'],
echo=not quiet,
critical=True)
return (stderr + stdout).strip()
# branch = subprocess.Popen(["git", "rev-parse", "--abbrev-ref", 'HEAD'], stdout=subprocess.PIPE).communicate()[0][:-1]
# if branch:
# return branch.decode('utf-8')
except Exception as e:
print(e)
pass
return '[UNKNOWN]'
def GetBranch(cls, quiet=True):
try:
cmd = ["git", "symbolic-ref", "--short", 'HEAD']
#stderr, stdout = cmd_output(["git", "rev-parse", "--abbrev-ref", 'HEAD', '--'], echo=not quiet, critical=True)
stderr, stdout = cmd_output(cmd, echo=not quiet, critical=True)
o = (stderr + stdout).decode('utf-8').strip()
if cls.OutputIsFatal(o):
log.error(_args2str(cmd))
return None
return o.strip()
except Exception as e:
print(e)
pass
return None
def IsDirty(cls, quiet=True):
try:
# branch = subprocess.Popen(['git', 'ls-files', '-m', '-o', '-d', '--exclude-standard'], stdout=subprocess.PIPE).communicate()[0][:-1]
# if branch:
stderr, stdout = cmd_output(
['git', 'ls-files', '-m', '-o', '-d', '--exclude-standard'],
echo=not quiet,
critical=True)
for line in (stderr + stdout).decode('utf-8').split('\n'):
line = line.strip()
if line != '': return True
return False
except Exception as e:
print(e)
pass
return None
def GetCommit(cls, ref='HEAD', short=True, quiet=True):
try:
addtl_flags = []
if short: addtl_flags.append('--short')
stdout, stderr = cmd_output(['git', 'rev-parse'] + addtl_flags +
[ref],
echo=not quiet,
critical=True)
return (stdout + stderr).decode('utf-8').strip()
# rev = subprocess.Popen(['git', 'rev-parse'] + addtl_flags + [ref], stdout=subprocess.PIPE).communicate()[0][:-1]
# if rev:
# return rev.decode('utf-8')
except Exception as e:
print(e)
pass
return '[UNKNOWN]'
def _resolveTagNoChdir(self, tag, quiet=None):
if quiet is None:
quiet = self.quiet
ret = cmd_output(
['git', 'rev-list', '-n', '1', 'refs/tags/{}'.format(tag)],
echo=not quiet,
env=self.noPasswordEnv)
if not ret:
return None
stdout, stderr = ret
for line in (stdout + stderr).decode('utf-8').split('\n'):
line = line.strip()
if line == '':
continue
if line.startswith('fatal:'):
log.error('[git] ' + line)
return None
return line.strip()
return None
def GetRemoteState(self, remote='origin', branch='master'):
with Chdir(self.path, quiet=self.quiet):
ret = cmd_output(['git', 'fetch', '-q'], echo=not self.quiet)
if not ret:
return False
stdout, stderr = ret
for line in (stdout + stderr).split('\n'):
line = line.strip()
if line == '':
continue
if line.startswith('fatal:'):
log.error('[git] ' + line)
return False
remoteinfo = Git.LSRemote(remote, branch)
if remoteinfo is None:
return False
ref = 'refs/heads/' + branch
if ref in remoteinfo:
self.remote_commit = remoteinfo[ref]
return True
def UpdateRemotes(self, remote=None, quiet=None):
if quiet is None:
quiet = self.quiet
if remote is not None:
self.remotes[remote] = self._syncRemote(remote, quiet=quiet)
return True
stdout, stderr = cmd_output(['git', 'remote', 'show'],
echo=not quiet,
env=self.noPasswordEnv)
for oline in (stdout + stderr).decode('utf-8').split('\n'):
line = oline.strip()
if not quiet or not quiet:
print(oline)
if line == '':
continue
if line.startswith('fatal:'):
log.error('[git] ' + line)
return False
self.remotes[line] = self._syncRemote(line, quiet=quiet)
return True
def _getRemoteInfo(self, remoteID):
'''
$ git remote show origin
* remote origin
Fetch URL: https://github.com/d3athrow/vgstation13.git
Push URL: https://github.com/d3athrow/vgstation13.git
HEAD branch: Bleeding-Edge
Remote branches:
Bleeding-Edge tracked
returns:
https://github.com/d3athrow/vgstation13.git
'''
stdout, stderr = cmd_output(['git', 'remote', 'show', remoteID],
echo=not self.quiet)
for line in (stdout + stderr).split('\n'):
line = line.strip()
components = line.split()
if line.startswith('Fetch URL:'):
# self.remotes[remoteID]=line[2]
return line[2]
def _syncRemote(self, remoteID, quiet=None):
'''
$ git remote show origin
* remote origin
Fetch URL: https://github.com/d3athrow/vgstation13.git
Push URL: https://github.com/d3athrow/vgstation13.git
HEAD branch: Bleeding-Edge
Remote branches:
Bleeding-Edge tracked
returns:
GitRemoteInfo()
'''
o = GitRemoteInfo()
o.id = remoteID
if quiet is None:
quiet = self.quiet
stdout, stderr = cmd_output(['git', 'remote', 'show', remoteID],
echo=not quiet,
env=self.noPasswordEnv)
in_branches = False
for oline in (stdout + stderr).decode('utf-8').split('\n'):
line = oline.strip()
if not quiet:
print(oline)
components = line.split()
if not in_branches:
if line.startswith('Fetch URL:'):
o.fetch_uri = components[2]
if line.startswith('Push URL:'):
o.push_uri = components[2]
if line.startswith('HEAD branch:'):
o.head_branch = components[2]
if line.startswith('Remote branches:'):
in_branches = True
else:
o.branches.append(line.split(' ')[0])
o.fetched = True
return o
def GetRemoteState(self, remote='origin', branch='HEAD', quiet=None):
if quiet is None:
quiet = self.quiet
with Chdir(self.path, quiet=self.quiet):
ret = cmd_output(
['git', 'fetch', '-q', '--all', '--prune', '--tags'],
echo=not quiet,
env=self.noPasswordEnv)
if not ret:
return False
stdout, stderr = ret
for line in (stdout + stderr).decode('utf-8').split('\n'):
# if not quiet:
# print(line)
line = line.strip()
if line == '':
continue
if line.startswith('fatal:'):
log.error('[git] ' + line)
return False
for _remote in self.remotes.values():
_remote.fetched = False
self.UpdateRemotes(remote=remote, quiet=quiet)
if branch == 'HEAD':
branch = self.remotes[remote].findBranch(branch)
remoteinfo = Git.LSRemote(remote, branch, quiet=quiet)
if remoteinfo is None:
return False
if branch == 'HEAD':
ref = 'HEAD'
elif '/' not in branch:
ref = 'refs/heads/' + branch
if ref in remoteinfo:
self.remote_commit = remoteinfo[ref]
return True
def _git(self, args, echo=False):
return cmd_output(['git'] + args, echo=echo, env=self.noPasswordEnv)
def build(self):
gitmodules = {}
with open(self.gitmodulesfile, 'r') as tomlf:
smid = None
for line in tomlf:
line = line.strip()
m = REG_SUBMODULE_SECTION.match(line)
if m is not None:
smid = m.group(1).strip()
gitmodules[smid] = {}
if '=' in line:
k, v = line.split('=', 2)
gitmodules[smid][k.strip()] = v.strip()
gitconfig = {}
with open(self.gitconfigfile, 'r') as tomlf:
smid = None
for line in tomlf:
line = line.strip()
#print(line)
m = REG_SUBMODULE_SECTION.match(line)
if m is not None:
smid = m.group(1).strip()
gitconfig[smid] = {}
if smid is not None and '=' in line:
#print(line)
k, v = line.split('=', 2)
gitconfig[smid][k.strip()] = v.strip()
'''
with open(self.gitmodulesfile + '.yml', 'w') as f:
yaml.dump(gitmodules, f, default_flow_style=False)
with open('.gitconfig.yml', 'w') as f:
yaml.dump(gitconfig, f, default_flow_style=False)
'''
for repoID, repoconf in gitconfig.items():
if repoID not in gitmodules.keys():
with log.warn('Submodule %s is present in .git/config but not .gitmodules!', repoID):
pathspec = repoconf.get('path', repoID)
path = os.path.abspath(pathspec)
tag = repoconf.get('tag', None)
branch = repoconf.get('branch', 'HEAD' if tag is None else None)
log.info('path = %s', pathspec)
for repoID, repoconf in gitmodules.items():
if repoID not in gitconfig.keys():
with log.warn('Submodule %s is present in .gitmodules but not .git/config!', repoID):
pathspec = repoconf.get('path', repoID)
path = os.path.abspath(pathspec)
tag = repoconf.get('tag', None)
branch = repoconf.get('branch', 'HEAD' if tag is None else None)
opts = []
if branch != 'HEAD':
opts += ['-b', branch]
log.info('path = %s', pathspec)
if os.path.isdir(path):
log.warn('Removing existing %s directory.', path)
shutil.rmtree(path)
cmd = ['git', 'submodule', 'add']+opts+['-f', '--name', repoID, '--', repoconf.get('url'), pathspec]
os_utils.cmd(cmd, critical=True, echo=self.should_echo_commands(), show_output=True)
#log.error('Would exec: %s', ' '.join(cmd))
for repoID, repoconf in gitmodules.items():
with log.info('Checking %s...', repoID):
pathspec = repoconf.get('path', repoID)
path = os.path.abspath(pathspec)
tag = repoconf.get('tag', None)
branch = repoconf.get('branch', 'HEAD' if tag is None else None)
if os.path.isdir(path):
desired_commit = ''
cmdline = ['git', 'ls-tree', Git.GetBranch(), pathspec]
stdout, stderr = os_utils.cmd_output(cmdline, echo=self.should_echo_commands(), critical=True)
skip_this = False
for line in (stdout+stderr).decode('utf-8').splitlines():
if line.startswith('error:') or line.startswith('fatal:'):
log.critical(line)
raise error.SubprocessThrewError(cmdline, line)
line,repoID = line.strip().split('\t')
_, _, desired_commit = line.split(' ')
if not skip_this:
with os_utils.Chdir(path, quiet=not self.should_echo_commands()):
cur_commit = Git.GetCommit(short=False, quiet=not self.should_echo_commands())
#log.info(desired_commit)
#log.info(cur_commit)
if cur_commit == desired_commit:
log.info('Commits are synced, skipping.')
continue
repo = GitRepository(path, origin_uri=repoconf['url'], submodule=True)
if repo.CheckForUpdates(branch=branch, quiet=False):
if os.path.isdir(path):
os_utils.cmd(['git', 'submodule', 'sync', '--', pathspec], critical=True, echo=self.should_echo_commands(), show_output=True)
os_utils.cmd(['git', 'submodule', 'update', '--init', '--recursive', pathspec], critical=True, echo=self.should_echo_commands(), show_output=True)
def _git(self, args, echo=False):
ret = cmd_output(['git'] + args, echo=echo)