def __init__(self, url, path, relpath, refspecs=None):
self.path = path
self.relpath = relpath
self.tools = Tools()
self.url = url
self.default_refspecs = [
'+refs/heads/*:refs/remotes/origin/*',
'+refs/tags/*:refs/remotes/origin/*',
]
self.refspecs = refspecs or []
self._tree_hash = {}
self._original_head = True
self._init_repo()
def upload(self, node, force=False):
if self._disabled:
return True
if not self._upload and not force:
return True
with self._cache.get_artifact(node) as artifact:
ftp = self._get_ftp()
if ftp is None:
return False
pathname = artifact.get_archive_path()
name = fs.path.basename(pathname)
size = Tools().file_size(pathname)
with log.progress("Uploading {0}".format(name), size, "B") as pbar:
with open(pathname, 'rb') as in_file:
if not catch(ftp.cwd, node.canonical_name):
if not catch(ftp.mkd, node.canonical_name):
return False
if not catch(ftp.cwd, node.canonical_name):
return False
catch(ftp.delete, name)
return catch(ftp.storbinary,
"STOR {filename}".format(filename=name),
in_file,
callback=lambda b: pbar.update(len(b)))
return False
def get_influence(self, task):
self.tools = Tools(task, task.joltdir)
try:
manifest_path = fs.path.join(task.joltdir, task.expand(self.path))
manifest = RepoManifest(task, manifest_path)
manifest.parse(fs.path.join(manifest_path, ".repo",
"manifest.xml"))
result = []
for project in sorted(manifest.projects, key=lambda p: p.name):
if self.include is not None and project.path_or_name not in self.include:
continue
if self.exclude is not None and project.path_or_name in self.exclude:
continue
with _git_repo_lock:
gip = _git_repos.get(project.path_or_name)
if gip is None:
gip = git.GitInfluenceProvider(project.path_or_name)
_git_repos[project.path_or_name] = gip
result.append(gip.get_influence(task))
return "\n".join(result)
except KeyError:
log.exception()
assert False, "failed to calculate hash influence for repo manifest at {0}".format(
self.path)
def write_tree(self):
tools = Tools()
with tools.cwd(self._git_folder()):
tools.copy("index", "jolt-index")
with tools.environ(GIT_INDEX_FILE=self._git_jolt_index()), tools.cwd(
self.path):
tree = tools.run(
"git -c core.safecrlf=false add -A && git write-tree",
output_on_error=True)
return tree
def get_influence(self, task):
tools = Tools(task, task.joltdir)
path = tools.expand_path(self.path)
git_abs = self._find_dotgit(path)
git_rel = git_abs[len(self.joltdir) + 1:]
relpath = path[len(git_abs) + 1:]
relpath = fs.as_posix(relpath) if relpath else relpath
if not fs.path.exists(path):
return "{0}/{1}: N/A".format(git_rel, relpath)
try:
git = new_git(None, git_abs, git_rel)
return "{0}/{1}: {2}".format(git_rel, relpath,
git.tree_hash(path=relpath or "/"))
except JoltCommandError as e:
stderr = "\n".join(e.stderr)
if "exists on disk, but not in" in stderr:
return "{0}/{1}: N/A".format(git_rel, relpath)
raise e
def run(self):
with open("default.joltxmanifest", "wb") as f:
f.write(self.body)
log.info("Manifest written")
tools = Tools()
for recipe in tools.glob("*.jolt"):
tools.unlink(recipe)
try:
jolt = self.selfdeploy()
config_file = config.get("amqp", "config", "")
if config_file:
config_file = "-c " + config_file
log.info("Running jolt")
tools.run(
"{} -vv {} build --worker --result result.joltxmanifest",
jolt,
config_file,
output_stdio=True)
except JoltCommandError as e:
self.response = ""
try:
manifest = JoltManifest()
try:
manifest.parse("result.joltxmanifest")
except Exception:
manifest.duration = "0"
manifest.result = "FAILED"
manifest.stdout = "\n".join(e.stdout)
manifest.stderr = "\n".join(e.stderr)
self.response = manifest.format()
except Exception:
log.exception()
log.error("Task failed")
except Exception:
log.exception()
self.response = ""
try:
manifest = JoltManifest()
try:
manifest.parse("result.joltxmanifest")
except Exception:
manifest.duration = "0"
manifest.result = "FAILED"
self.response = manifest.format()
except Exception:
log.exception()
log.error("Task failed")
else:
self.response = ""
try:
manifest = JoltManifest()
try:
manifest.parse("result.joltxmanifest")
except Exception:
manifest.duration = "0"
manifest.result = "SUCCESS"
self.response = manifest.format()
except Exception:
log.exception()
log.info("Task succeeded")
utils.call_and_catch(tools.unlink, "result.joltxmanifest")
self.consumer.add_on_job_completed_callback(self)
def selfdeploy(self):
""" Installs the correct version of Jolt as specified in execution request. """
tools = Tools()
manifest = JoltManifest()
try:
manifest.parse()
ident = manifest.get_parameter("jolt_identity")
url = manifest.get_parameter("jolt_url")
if not ident or not url:
return "jolt"
requires = manifest.get_parameter("jolt_requires")
log.info("Jolt version: {}", ident)
src = "build/selfdeploy/{}/src".format(ident)
env = "build/selfdeploy/{}/env".format(ident)
if not fs.path.exists(env):
try:
fs.makedirs(src)
tools.run("curl {} | tar zx -C {}", url, src)
tools.run("virtualenv {}", env)
tools.run(". {}/bin/activate && pip install -e {}",
env, src)
if requires:
tools.run(
". {}/bin/activate && pip install {}", env,
requires)
except Exception as e:
tools.rmtree("build/selfdeploy/{}",
ident,
ignore_errors=True)
raise e
return ". {}/bin/activate && jolt".format(env)
except Exception as e:
log.exception()
raise e
def is_influenced_by(self, task, path):
tools = Tools(task, task.joltdir)
gitpath = tools.expand_path(self.path)
return fs.is_relative_to(path, gitpath)
class GitRepository(object):
def __init__(self, url, path, relpath, refspecs=None):
self.path = path
self.relpath = relpath
self.tools = Tools()
self.url = url
self.default_refspecs = [
'+refs/heads/*:refs/remotes/origin/*',
'+refs/tags/*:refs/remotes/origin/*',
]
self.refspecs = refspecs or []
self._tree_hash = {}
self._original_head = True
self._init_repo()
def _init_repo(self):
self.repository = pygit2.Repository(self.path) if os.path.exists(
self._git_folder()) else None
@utils.cached.instance
def _git_folder(self):
return fs.path.join(self.path, ".git")
@utils.cached.instance
def _git_index(self):
return fs.path.join(self.path, ".git", "index")
@utils.cached.instance
def _git_jolt_index(self):
return fs.path.join(self.path, ".git", "jolt-index")
def is_cloned(self):
return fs.path.exists(self._git_folder())
def is_indexed(self):
return fs.path.exists(self._git_index())
def clone(self):
log.info("Cloning into {0}", self.path)
if fs.path.exists(self.path):
with self.tools.cwd(self.path):
self.tools.run(
"git init && git remote add origin {} && git fetch",
self.url,
output_on_error=True)
else:
self.tools.run("git clone {0} {1}",
self.url,
self.path,
output_on_error=True)
raise_error_if(not fs.path.exists(self._git_folder()),
"git: failed to clone repository '{0}'", self.relpath)
self._init_repo()
@utils.cached.instance
def diff_unchecked(self):
if not self.is_indexed():
return ""
# Build the jolt index file
self.write_tree()
# Diff against the jolt index
with self.tools.environ(
GIT_INDEX_FILE=self._git_jolt_index()), self.tools.cwd(
self.path):
return self.tools.run("git diff --binary --no-ext-diff HEAD ./",
output_on_error=True,
output_rstrip=False)
def diff(self):
diff = self.diff_unchecked()
dlim = config.getsize("git", "maxdiffsize", "1M")
raise_error_if(
len(diff) > dlim,
"git patch for '{}' exceeds configured size limit of {} bytes - actual size {}"
.format(self.path, dlim, len(diff)))
return diff
def patch(self, patch):
if not patch:
return
with self.tools.cwd(self.path), self.tools.tmpdir("git") as t:
patchfile = fs.path.join(t.path, "jolt.diff")
with open(patchfile, "wb") as f:
f.write(patch.encode())
log.info("Applying patch to {0}", self.path)
self.tools.run("git apply --whitespace=nowarn {patchfile}",
patchfile=patchfile)
def head(self):
if not self.is_cloned():
return None
return str(self.repository.head.target)
def is_head(self, rev):
return self.is_indexed() and self.head() == rev
def is_original_head(self):
return self._original_head
def is_valid_sha(self, rev):
return re.match(r"[0-9a-f]{40}", rev)
def rev_parse(self, rev):
if self.is_valid_sha(rev):
return rev
with self.tools.cwd(self.path):
try:
commit = self.repository.revparse_single(rev)
except KeyError:
self.fetch()
try:
commit = self.repository.revparse_single(rev)
except Exception:
raise_error("invalid git reference: {}", rev)
try:
return str(commit.id)
except Exception:
return str(commit)
@utils.cached.instance
def write_tree(self):
tools = Tools()
with tools.cwd(self._git_folder()):
tools.copy("index", "jolt-index")
with tools.environ(GIT_INDEX_FILE=self._git_jolt_index()), tools.cwd(
self.path):
tree = tools.run(
"git -c core.safecrlf=false add -A && git write-tree",
output_on_error=True)
return tree
def tree_hash(self, sha=None, path="/"):
# When sha is None, the caller want the tree hash of the repository's
# current workspace state. If no checkout has been made, that would be the
# tree that was written upon initialization of the repository as it
# includes any uncommitted changes. If a checkout has been made since
# the repo was initialized, make this an explicit request for the current
# head - there can be no local changes.
if sha is None:
if self.is_original_head():
tree = self.repository.get(self.write_tree())
else:
sha = self.head()
path = fs.path.normpath(path)
full_path = fs.path.join(self.path, path) if path != "/" else self.path
# Lookup tree hash value in cache
value = self._tree_hash.get((full_path, sha))
if value is not None:
return value
# Translate explicit sha to tree
if sha is not None:
commit = self.rev_parse(sha)
obj = self.repository.get(commit)
try:
tree = obj.tree
except AttributeError:
tree = obj.get_object().tree
# Traverse tree from root to requested path
if path != "/":
tree = tree[fs.as_posix(path)]
# Update tree hash cache
self._tree_hash[(full_path, sha)] = value = tree.id
return value
def clean(self):
with self.tools.cwd(self.path):
return self.tools.run("git clean -dfx", output_on_error=True)
def reset(self):
with self.tools.cwd(self.path):
return self.tools.run("git reset --hard", output_on_error=True)
def fetch(self):
refspec = " ".join(self.default_refspecs + self.refspecs)
with self.tools.cwd(self.path):
log.info("Fetching {0} from {1}", refspec or 'commits', self.url)
self.tools.run("git fetch {url} {refspec}",
url=self.url,
refspec=refspec or '',
output_on_error=True)
def checkout(self, rev):
log.info("Checking out {0} in {1}", rev, self.path)
with self.tools.cwd(self.path):
try:
return self.tools.run("git checkout -f {rev}",
rev=rev,
output=False)
except Exception:
self.fetch()
return self.tools.run("git checkout -f {rev}",
rev=rev,
output_on_error=True)
self._original_head = False