def __init__(self, commit_id, tree, revert=False):
self._msg_id = commit_id
if revert:
self._tree_id = git.rev_parse('%s~1' % commit_id, tree=tree)
else:
self._tree_id = git.rev_parse('%s' % commit_id, tree=tree)
self._revert = revert
self._tree = tree
def __init__(self, name=None, commit=None, create=False):
"""
Construct a blueprint in the new format in a backwards-compatible
manner.
"""
self.name = name
self._commit = commit
# Create a new blueprint object and populate it based on this server.
if create:
super(Blueprint, self).__init__()
import backend
for funcname in backend.__all__:
getattr(backend, funcname)(self)
import services
services.services(self)
# Create a blueprint from a Git repository.
elif name is not None:
git.init()
if self._commit is None:
self._commit = git.rev_parse('refs/heads/{0}'.format(name))
if self._commit is None:
raise NotFoundError(name)
tree = git.tree(self._commit)
blob = git.blob(tree, 'blueprint.json')
content = git.content(blob)
super(Blueprint, self).__init__(**json.loads(content))
# Create an empty blueprint object to be filled in later.
else:
super(Blueprint, self).__init__()
def detect_change(args):
# Caches the line-endings for the file in a revision.
le_cache = {}
def get_le(rev):
try:
return le_cache[rev]
except KeyError:
pass
try:
result = get_line_endings(git.get_file(rev, args.file))
except git.DoesNotExist as e:
result = None
le_cache[rev] = result
return result
try:
args.rev = git.rev_parse(args.rev)
commits = list(git.rev_list(args.rev))
for index, rev in enumerate(commits):
print('\r {}/{} ({})'.format(index, len(commits), rev), end='')
revs = [rev] + git.parents(rev)
le = [get_le(x) for x in revs]
changed_indices = [i for i in range(1, len(le)) if le[i] and le[i] != le[0]]
if changed_indices:
print('\r', end='')
for i in changed_indices:
print('{}..{} ({}..{})'.format(revs[i], revs[0], le[i].upper(), le[0].upper()), end='')
if len(revs) > 2:
print(' merge-commit', end='')
print()
finally:
print('\r')
def __init__(self, name=None, commit=None, create=False):
"""
Construct a blueprint in the new format in a backwards-compatible
manner.
"""
self.name = name
self._commit = commit
# Create a new blueprint object and populate it based on this server.
if create:
super(Blueprint, self).__init__()
for funcname in backend.__all__:
getattr(backend, funcname)(self)
# Create a blueprint from a Git repository.
elif name is not None:
git.init()
if self._commit is None:
self._commit = git.rev_parse('refs/heads/%s' % (name))
if self._commit is None:
raise KeyError(name)
tree = git.tree(self._commit)
blob = git.blob(tree, 'blueprint.json')
content = git.content(blob)
super(Blueprint, self).__init__(**json.loads(content))
# Create an empty blueprint object to be filled in later.
else:
super(Blueprint, self).__init__()
def checkout(cls, name, commit=None):
git.init()
if commit is None:
commit = git.rev_parse('refs/heads/{0}'.format(name))
if commit is None:
raise NotFoundError(name)
tree = git.tree(commit)
blob = git.blob(tree, 'blueprint.json')
content = git.content(blob)
return cls(name, commit, **json.loads(content))
def commit(self, message=''):
"""
Create a new revision of this blueprint in the local Git repository.
Include the blueprint JSON and any source archives referenced by
the JSON.
"""
git.init()
refname = 'refs/heads/{0}'.format(self.name)
parent = git.rev_parse(refname)
# Start with an empty index every time. Specifically, clear out
# source tarballs from the parent commit.
if parent is not None:
for mode, type, sha, pathname in git.ls_tree(git.tree(parent)):
git.git('update-index', '--force-remove', pathname)
# Add `blueprint.json` to the index.
f = open('blueprint.json', 'w')
f.write(self.dumps())
f.close()
git.git('update-index', '--add', os.path.abspath('blueprint.json'))
# Add source tarballs to the index.
for filename in self.sources.itervalues():
git.git('update-index', '--add', os.path.abspath(filename))
# Add `/etc/blueprintignore` and `~/.blueprintignore` to the index.
# Since adding extra syntax to this file, it no longer makes sense
# to store it as `.gitignore`.
f = open('blueprintignore', 'w')
for pathname in ('/etc/blueprintignore',
os.path.expanduser('~/.blueprintignore')):
try:
f.write(open(pathname).read())
except IOError:
pass
f.close()
git.git('update-index', '--add', os.path.abspath('blueprintignore'))
# Write the index to Git's object store.
tree = git.write_tree()
# Write the commit and update the tip of the branch.
self._commit = git.commit_tree(tree, message, parent)
git.git('update-ref', refname, self._commit)
def commit(self, message=''):
"""
Create a new revision of this blueprint in the local Git repository.
Include the blueprint JSON and any source archives referenced by
the JSON.
"""
git.init()
refname = 'refs/heads/{0}'.format(self.name)
parent = git.rev_parse(refname)
# Start with an empty index every time. Specifically, clear out
# source tarballs from the parent commit.
if parent is not None:
for mode, type, sha, pathname in git.ls_tree(git.tree(parent)):
git.git('update-index', '--force-remove', pathname)
# Add `blueprint.json` to the index.
f = open('blueprint.json', 'w')
f.write(self.dumps())
f.close()
git.git('update-index', '--add', os.path.abspath('blueprint.json'))
# Add source tarballs to the index.
for filename in self.sources.itervalues():
git.git('update-index', '--add', os.path.abspath(filename))
# Add `/etc/blueprintignore` and `~/.blueprintignore` to the index.
# Since adding extra syntax to this file, it no longer makes sense
# to store it as `.gitignore`.
f = open('blueprintignore', 'w')
for pathname in ('/etc/blueprintignore',
os.path.expanduser('~/.blueprintignore')):
try:
f.write(open(pathname).read())
except IOError:
pass
f.close()
git.git('update-index', '--add', os.path.abspath('blueprintignore'))
# Write the index to Git's object store.
tree = git.write_tree()
# Write the commit and update the tip of the branch.
self._commit = git.commit_tree(tree, message, parent)
git.git('update-ref', refname, self._commit)
def _find_commits(self, branch, start_commit, input_work_dir,
output_work_dir):
# If somebody manually modified the output tree without pointing to an
# input tree commit (which really should happen only in the case of
# ChromeOS where we may merge upstream) then search for the original
# commit pointer also in the parent, etc. of the top-most output commit.
last_commit = None
for back in range(self.max_parents):
last_commit = git.get_note(self.notes_branch,
'%s~%d' % (start_commit, back),
tree=output_work_dir)
if last_commit is not None:
last_commit = last_commit.strip()
self.debug(
'Found note on %s~%d indicating last checked commit is %s'
% (start_commit, back, last_commit))
break
try:
msg = git.commit_message('%s~%d' % (start_commit, back),
tree=output_work_dir)
except git.GitError:
msg = ''
match = self._id_re.search(msg)
if match:
last_commit = match.group(1)
break
if last_commit:
return self._log_commits(last_commit, 'origin/' + branch,
input_work_dir)
# Note that this is also important if working on a new branch, where
# we get here through self._need_to_mirror() with start_commit being
# just the branch name - so above we'll have tried something like
# new_branch~0, new_branch~1 etc. none of which can exists. So now
# this will give up and tell us to create just the single commit,
# which at least is some work to do so we can sort it out later in
# self._prepare_output_git() which detects and bases a new branch
# properly.
commit_id = git.rev_parse('origin/' + branch, tree=input_work_dir)
return [Commit(commit_id, input_work_dir)]
def _handle_submodule(self, branch, outer_commit, submodule_path,
prev_commit, cur_commit, input_work_dir,
output_work_dir, start_commit, new_branch):
submodule_tree = os.path.join(input_work_dir, submodule_path)
commits = self._log_commits(prev_commit, cur_commit, submodule_tree)
for commit in commits:
# ok ... if any fails just abandon the whole thing - could be better, but ...
if not self._create_output(branch,
outer_commit,
input_work_dir,
output_work_dir,
start_commit,
new_branch,
submodules={submodule_path: commit}):
return []
git.add('.', tree=output_work_dir)
if self.is_modified(tree=output_work_dir):
self._commit_modified(commit,
submodule_tree,
output_work_dir,
None,
add_id=False)
return [git.rev_parse('HEAD', tree=output_work_dir)]
def _check_new_branch(self, branch, input_work_dir, output_work_dir,
output_reference):
# try to find merge-base in input tree,
# assumes master branch is always mirrored
#
# this handles the case of having created a new branch,
# and asking for that to be mirrored into the prune tree.
base_id = git.merge_base('origin/' + self.master,
'origin/' + branch,
tree=input_work_dir)
git.clone(output_reference, output_work_dir, options=['-q'])
git.set_origin_url(self._output_tree, output_work_dir)
# try to find the merge-base or its parent/grandparent/... in the
# output tree - since it should have been branched from the master
# (or in any case we look at the merge-base between master and it)
# this should exist - HOWEVER: some commits might not be mirrored,
# so look for the *parent [with a reasonable limit]
for offset in range(0, self.max_parents):
search = git.rev_parse(rev='%s~%d' % (base_id, offset),
tree=input_work_dir)
self.debug('search for %s~%d=%s:' % (base_id, offset, search))
grep = '%s: %s' % (self._commit_id_prefix, search)
out_commits = git.log(options=[
'--grep', grep, '--format=format:%H',
'origin/' + self.output_branch_name(self.master)
],
tree=output_work_dir)
out_commits = out_commits.split()
if not out_commits or len(out_commits) > 1:
self.debug('not found')
continue
start_commit = out_commits[0]
self.debug('found at %s' % start_commit)
return start_commit
raise Exception('failed to find parent/grandparent/...')
def _mirror_one(self, branch, input_work_dir, output_reference):
self.debug('*********** start work for branch %s -> %s' % (
branch,
self.output_branch_name(branch),
))
if not self._need_to_mirror(branch, input_work_dir, output_reference):
return True
with tempdir.tempdir() as tmpdir:
output_work_dir = os.path.join(tmpdir, 'output')
start_commit, new_branch = self._prepare_output_git(
branch, input_work_dir, output_work_dir, output_reference)
commits = self._find_commits(branch, start_commit, input_work_dir,
output_work_dir)
os.chdir(tmpdir)
committed_anything = False
last_failure = None
try:
# In this case we're already done, the loop below would be skipped
# completely, but we can't even calculate the (unnecessary)
# prep_prev_commit and similar, nor do we really have to check out
# the code to generate nothing. However, if it's a new branch we
# may have to push it out, so go through the "finally:" segment of
# the code (and hence have the return statement within the "try:"
# block.
# The output tree is correct since self._prepare_output_git() will
# leave it at the commit it wanted to start generating from (even
# if there's nothing to generate, it doesn't consider that.)
if len(commits) == 0:
return True
prep_prev_commit = Commit(
git.rev_parse(commits[0].tree_id + '^',
tree=input_work_dir), input_work_dir)
self._checkout(prep_prev_commit.tree_id, input_work_dir)
submodules = self._submodule_status(input_work_dir)
for commit in commits:
prev_commit = prep_prev_commit
prep_prev_commit = commit
if not self._create_output(branch, commit, input_work_dir,
output_work_dir, start_commit,
new_branch):
if last_failure is None:
last_failure = commit
continue
if last_failure:
last_failure_shortlog = git.shortlog(
last_failure, commit)
else:
last_failure_shortlog = None
git.add('.', tree=output_work_dir)
prev_submodules = submodules
submodules = self._submodule_status(input_work_dir)
if self.is_modified(tree=output_work_dir):
parents = [git.rev_parse('HEAD', tree=output_work_dir)]
tree_id = git.write_tree(tree=output_work_dir)
for s in submodules:
if not s in prev_submodules:
continue
if prev_submodules[s] != submodules[s]:
parents += self._handle_submodule(
branch, prev_commit, s, prev_submodules[s],
submodules[s], input_work_dir,
output_work_dir, start_commit, new_branch)
self._commit_modified(commit, input_work_dir,
output_work_dir,
last_failure_shortlog, tree_id,
parents)
committed_anything = True
elif new_branch and self.always_commit_new_branch:
self._commit_new_branch(commit, output_work_dir)
committed_anything = True
new_branch = False
last_failure = None
git.set_note(self.notes_branch,
'HEAD',
commit.tree_id,
tree=output_work_dir,
env=self._commit_env())
except Abort:
return False
finally:
# if necessary, push to the server from the output_work_dir
git.set_origin_url(self._output_tree, gitdir=output_work_dir)
if committed_anything or new_branch:
git.push(opts=[
'-q', 'origin',
'HEAD:' + self.output_branch_name(branch)
],
tree=output_work_dir)
git.push(opts=[
'-q', '-f', 'origin', 'refs/notes/' + self.notes_branch
],
tree=output_work_dir)
return True
import io
import os
import requests
import types
import pytest
import problem
from problem import Problem
from problem import Difficulty
repo = Repo()
assert (repo.bare == False)
git = repo.git
initial_branch = git.rev_parse('--abbrev-ref', 'HEAD')
# HEAD indicates we're not on a branch
if initial_branch == 'HEAD':
initial_branch = repo.head.commit.hexsha
def test_init_happy_path():
args = Problem.default_args()
args.name = '3sum'
p = Problem(args)
# misc attributes
assert (p._verbose == False)
# leetcode attributes
def walk(b, choose):
"""
Given a function for choosing a `Blueprint` object (based typically on
the result of a `raw_input` call within the `choose` function), populate
one or more `Blueprint`s closed into `choose`.
"""
def file(pathname, f):
print(pathname)
b_chosen = choose()
if b_chosen is None:
return
b_chosen.add_file(pathname, **f)
def package(manager, package, version):
print('{0} {1} {2}'.format(manager, package, version))
b_chosen = choose()
if b_chosen is None:
return
b_chosen.add_package(manager, package, version)
def service(manager, service):
print('{0} {1}'.format(manager, service))
b_chosen = choose()
if b_chosen is None:
return
b_chosen.add_service(manager, service)
def service_file(manager, service, pathname):
b_chosen.add_service_file(manager, service, pathname)
walklib.walk_service_files(b_chosen,
manager,
service,
service_file=service_file)
def service_package(manager, service, package_manager, package):
b_chosen.add_service_package(manager, service, package_manager,
package)
walklib.walk_service_packages(b_chosen,
manager,
service,
service_package=service_package)
def service_source(manager, service, dirname):
b_chosen.add_service_source(manager, service, dirname)
walklib.walk_service_sources(b_chosen,
manager,
service,
service_source=service_source)
commit = git.rev_parse(b.name)
tree = git.tree(commit)
def source(dirname, filename, gen_content, url):
if url is not None:
print('{0} {1}'.format(dirname, url))
elif gen_content is not None:
blob = git.blob(tree, filename)
git.cat_file(blob, filename)
print('{0} {1}'.format(dirname, filename))
b_chosen = choose()
if b_chosen is None:
return
b_chosen.add_source(dirname, filename)
b.walk(file=file, package=package, service=service, source=source)
def _check_new_branch(self, branch, input_work_dir, output_work_dir,
output_reference):
# This handles the case of having created a new branch,
# and asking for that to be mirrored into the prune tree.
#
# Try to find a starting point in the input tree. We assume
# this will basically always succeed, since the master branch
# is always mirrored, we _should_ find something (but can fail
# if somebody created a branch without a merge-base.)
# unfortunately we now need to do this first, to sort out which
# branches we already know in the output
git.clone(output_reference, output_work_dir, options=['-q'])
self.debug("trying to find starting point for new branch %s" % branch)
candidate_branches = []
for other in self._branches:
if other == branch:
continue
out_branch = 'origin/' + self.output_branch_name(other)
try:
git.rev_parse(out_branch, tree=output_work_dir)
except:
self.debug(" branch %s doesn't exist in output %s (yet)" %
(out_branch, output_work_dir))
continue
candidate_branches.append(other)
potential_merge_bases = []
for other in candidate_branches:
try:
base = git.merge_base('origin/' + other,
'origin/' + branch,
tree=input_work_dir)
potential_merge_bases.append(base)
self.debug(" base to %s is %s" % (other, base))
except git.GitError:
self.debug(" no base to %s" % (other, ))
bases = git.independent_commits(potential_merge_bases,
tree=input_work_dir)
self.debug("found starting points %s" % (", ".join(bases)))
assert len(bases) == 1, "No single merge base found: %r" % bases
base_id = bases[0]
base_branch = None
for other in candidate_branches:
if git.merge_base('origin/' + other, base_id,
tree=input_work_dir) == base_id:
base_branch = 'origin/' + self.output_branch_name(other)
break
assert base_branch, "This shouldn't happen, found no base branch?!"
# try to find the merge-base or its parent/grandparent/... in the
# output tree - since it should have been branched from the master
# (or in any case we look at the merge-base between master and it)
# this should exist - HOWEVER: some commits might not be mirrored,
# so look for the *parent [with a reasonable limit]
for offset in range(0, self.max_parents):
search = git.rev_parse(rev='%s~%d' % (base_id, offset),
tree=input_work_dir)
self.debug('search for %s~%d=%s:' % (base_id, offset, search))
grep = '%s: %s' % (self._commit_id_prefix, search)
out_commits = git.log(
options=['--grep', grep, '--format=format:%H', base_branch],
tree=output_work_dir)
out_commits = out_commits.split()
if not out_commits or len(out_commits) > 1:
self.debug('not found')
continue
start_commit = out_commits[0]
self.debug('found at %s' % start_commit)
return start_commit
raise Exception('failed to find parent/grandparent/...')
def walk(b, choose):
"""
Given a function for choosing a `Blueprint` object (based typically on
the result of a `raw_input` call within the `choose` function), populate
one or more `Blueprint`s closed into `choose`.
"""
def file(pathname, f):
print(pathname)
b_chosen = choose()
if b_chosen is None:
return
b_chosen.add_file(pathname, **f)
def package(manager, package, version):
print('{0} {1} {2}'.format(manager, package, version))
b_chosen = choose()
if b_chosen is None:
return
b_chosen.add_package(manager, package, version)
def service(manager, service):
print('{0} {1}'.format(manager, service))
b_chosen = choose()
if b_chosen is None:
return
b_chosen.add_service(manager, service)
def service_file(manager, service, pathname):
b_chosen.add_service_file(manager, service, pathname)
walklib.walk_service_files(b_chosen,
manager,
service,
service_file=service_file)
def service_package(manager, service, package_manager, package):
b_chosen.add_service_package(manager,
service,
package_manager,
package)
walklib.walk_service_packages(b_chosen,
manager,
service,
service_package=service_package)
def service_source(manager, service, dirname):
b_chosen.add_service_source(manager, service, dirname)
walklib.walk_service_sources(b_chosen,
manager,
service,
service_source=service_source)
commit = git.rev_parse(b.name)
tree = git.tree(commit)
def source(dirname, filename, gen_content, url):
if url is not None:
print('{0} {1}'.format(dirname, url))
elif gen_content is not None:
blob = git.blob(tree, filename)
git.cat_file(blob, filename)
print('{0} {1}'.format(dirname, filename))
b_chosen = choose()
if b_chosen is None:
return
b_chosen.add_source(dirname, filename)
b.walk(file=file, package=package, service=service, source=source)
