def start_get_hash(config, github, config_file):
repo_owner = config['repo_owner']
for team in config['teams']:
repo_name = config['teams'][team]['repo_name']
if repo_name == '-':
continue
print('[*] Get the commit hash of %s repo.' % repo_name)
bug_branches = config['teams'][team]['bug_branches']
clone(repo_owner, repo_name)
branches = bug_branches if len(bug_branches) > 0 \
else list_branches(repo_name)
if "master" in branches:
branches.remove("master") # Do not consider master branch
for branch in branches:
checkout(repo_name, branch)
hash = get_latest_commit_hash(repo_name, int(time.time()), branch)
config['teams'][team][branch] = hash
rmdir(repo_name)
with open(config_file, 'w') as outfile:
json.dump(config, outfile, indent=4)
print ('[*] Successfully write in %s' % config_file)
return
def release(self, tag):
logging.info("Doing to release %s:%s", self.__repos, tag)
try:
source_path = os.path.join(config.build_root, self.__repos)
git.checkout(self.__repos, tag, source_path)
build.make(source_path)
config_file = configparser.ConfigParser()
config_file.read(os.path.join(source_path, ".tinyci"))
release_id = github.addRelease(self.__repos, tag)
for section_name, section in filter(
lambda n: n[0].startswith("build-") or n[0] == "build",
config_file.items()):
for artifact in section.get("artifacts",
"").strip().split("\n"):
filename = os.path.join(source_path,
section.get("directory", ""),
artifact)
if os.path.isfile(filename):
name = os.path.splitext(os.path.basename(artifact))
name = "%s_%s%s" % (name[0], tag, name[1])
if len(section_name) > 6:
name = "%s_%s" % (section_name[6:].capitalize(),
name)
github.addReleaseAsset(self.__repos,
release_id,
filename,
name=name)
elif filename != "":
logging.warning(
"Missing artifact after release build: %s",
artifact)
github.publishRelease(self.__repos, release_id)
except Exception as e:
logging.exception("Exception %s while trying to release", e)
logging.info("Finished release for %s:%s", self.__repos, tag)
def start_service(service_dir, branch, container_name, flag_str, log=None):
log= print_and_log("[*] Starting service from %s (branch '%s')" % \
(service_dir, branch), log)
checkout(service_dir, branch)
# Update flag file
flag_path = os.path.join(service_dir, "flag") # Assumption in template
if not os.path.isfile(flag_path):
log = print_and_log("[*] 'flag' file not found in %s" % service_dir,
log)
return False, log
with open(flag_path, "w") as flag_file:
flag_file.write(flag_str)
# Run the service
script = os.path.join(base_dir(), "setup_service.sh")
cmdline = \
"%s %s %d %d" % (script, container_name, SERVICE_PORT, SERVICE_PORT)
output, err, e = run_command(cmdline, service_dir)
if e != 0:
log = print_and_log("[*] Failed to start service", log)
log = print_and_log(err, log)
log = print_and_log("==========================", log)
return False, log
if log is not None:
log = log + output
log = print_and_log("[*] Started service successfully", log)
return True, log
def verify_injection(team, config_file):
config = load_config(config_file)
timeout = config["exploit_timeout"]["injection_phase"]
repo_owner = config['repo_owner']
repo_name = config['teams'][team]['repo_name']
bug_branches = config['teams'][team]['bug_branches']
clone(repo_owner, repo_name)
branches = bug_branches if len(bug_branches) > 0 \
else list_branches(repo_name)
if "master" in branches:
branches.remove("master") # master branch is not verification target
for branch in branches:
checkout(repo_name, branch)
exploit_dir = get_exploit_dir(repo_name, branch, config, team)
bug_branch_result, _ = \
verify_exploit(exploit_dir, repo_name, branch, timeout, config)
checkout(repo_name, "master")
master_result, _ = \
verify_exploit(exploit_dir, repo_name, "master", timeout, config)
rmdir(exploit_dir)
if master_result == False and bug_branch_result == True:
print('[*] Successflly verified branch "%s".' % branch)
elif bug_branch_result == True:
print ('[*] Exploit for branch "%s" works, but it also works on ' \
'master branch, which indicates some error.' % branch)
sys.exit()
else:
print('[*] Failed to verify exploit in branch "%s".' % branch)
sys.exit()
rmdir(repo_name)
def test_commit_solution(monkeypatch):
my_stdin = '0\n100\nO(1)\n0\n100\nO(1)\n'
try:
git.branch('-D', 'issue/42/foo-bar')
except:
pass
monkeypatch.setattr('sys.stdin', io.StringIO(my_stdin))
p = problem.new_problem_args(
'foo-bar', 42, 3,
'class Solution {\npublic:\nint fooBar() {\nreturn 0;\n}\n};\n')
try:
git.add('foo-bar.cpp')
git.add('foo-bar-test.cpp')
problem.commit_solution()
except Exception as e:
git.checkout(initial_branch)
git.branch('-D', p._branch)
raise e
git.checkout(initial_branch)
git.branch('-D', p._branch)
def merge(source_branch, dest_branch):
""" Performs the squash-merge. """
if not git.check_branch_exists(source_branch):
return fatal('Source branch does not exist.')
if not git.check_branch_exists(dest_branch):
return fatal('Destination branch does not exist.')
if not git.is_clean_workdir():
return fatal('Current working copy should be clean in order to '
'perform merge.')
git.checkout(dest_branch)
merge_commit = _get_previous_merge_commit(source_branch, dest_branch)
if merge_commit:
print('Branch {source} has already been merged into {dest}. '
'Deleting the previous merge commit.'
''.format(source=source_branch, dest=dest_branch))
_delete_commit(merge_commit)
res = git.merge_squash(source_branch)
if res.status_code != 0:
return fatal('Automatic merge failed.\n'
'Run `git status` to see the conflicts.\n'
'Run `git reset --merge` to abort merge.')
# FIXME currently assumes that it is run from the repo root.
git.commit('--file=.git/SQUASH_MSG')
print('Merged {source} into {dest}.'
''.format(source=source_branch, dest=dest_branch))
def _git_merge(branch, local_branches):
if branch in local_branches:
git.remove_branch(branch)
git.checkout(branch)
git_master_head_local_done = git.get_head("remotes/origin/master")
git_remote_head_local_done = git.get_head(branch)
git.get_status()
git_merge_status = git.merge()
git_branch_head_merged = git.get_head(branch)
# sql_info, bsh_info, config_info = git.get_diff()
git_sql_info, git_bsh_info, git_config_info = None, None, None
dbcon.execute(
"""update branch set git_master_head_local_done=?, git_remote_head_local_done=?, git_branch_head_merged=?, git_merge_status=?, git_sql_info=?, git_bsh_info=?, git_config_info=?, git_last_update_time=? where branch=?;""",
(
git_master_head_local_done,
git_remote_head_local_done,
git_branch_head_merged,
git_merge_status,
git_sql_info,
git_bsh_info,
git_config_info,
int(time.time()),
branch,
),
)
def tag_all_remote_branches(b_tag_after_update, repo_abs_path, repo):
"""
Tag all remote branches with timestamps and branch names
"""
# preserve current status
# current_section_branch = git.get_current_branch()
# probably section path
section_path = os.getcwd()
if b_tag_after_update:
# need to obtain the branch list in the repository
os.chdir(repo_abs_path)
# preserve repository status
current_repo_branch = git.get_current_branch()
# branch name loop
for repo_branch in git.get_remote_branch_list():
# A remote branch would be like : remote_name/branch_name/##
tag_stamp(b_tag_after_update,
repo_abs_path,
repo,
branch=repo_branch,
commit=repo_branch)
# restore repository branch
git.checkout(current_repo_branch)
if 'master' != git.get_current_branch().strip():
print("branch = {branch}, repo path = {path}".format(
branch=git.get_current_branch(), path=repo_abs_path))
# return to section path
os.chdir(section_path)
def crowdin_upload_sources(repository, new_files):
before_upload = get_crowdin_file_info(repository)
for file in new_files:
git.checkout(file)
extension = file[file.rfind('.'):]
if extension == '.md' or extension == '.markdown':
_pandoc(file, file, '--from=gfm', '--to=gfm', '--wrap=none')
fix_product_name_tokens(file)
df = pd.read_csv('%s/ignore.csv' % initial_dir)
ignore_files = set(
df[df['repository'] == repository.github.upstream]['file'].values)
upload_files = [file for file in new_files if file not in ignore_files]
if len(upload_files) > 0:
configure_crowdin(repository, upload_files)
_crowdin('upload', 'sources')
for file in new_files:
git.checkout(file)
if len(upload_files) > 0:
after_upload = get_crowdin_file_info(repository)
else:
after_upload = before_upload
return before_upload, after_upload
def create_new_branch(new_branch):
# Delete it if it exists, not the most elegant solution
try:
git.branch('-D', new_branch)
except BaseException:
pass
print(f'Creating a branch with the new info')
git.checkout('-b', new_branch, SRCPATH)
def main():
usage = "usage: %prog [options]"
parser = optparse.OptionParser(usage)
parser.add_option("-m", "--merge-master", dest="merge_master",
action="store_true",
default=False,
help="Merges the latest master into the current branch")
parser.add_option("-B", "--merge-branch", dest="merge_branch",
action="store_true",
default=False,
help="Merge the current branch into master; forces -m")
options, args = parser.parse_args()
repo=Repo(os.getcwd())
git = repo.git
if not options.merge_master and not options.merge_branch:
parser.error('Must choose one-- try -m or -B')
# Merging branch requires latest merged master
if options.merge_branch:
options.merge_master = True
if options.merge_master:
output=repo.git.status()
print output
match = re.search('On branch ([^\s]*)', output)
print match
branch = None
if match is None:
raise Exception('Could not get status')
elif match.group(1) == 'master':
raise Exception('You must be in the branch that you want to merge, not master')
else:
branch = match.group(1)
logging.info('In branch %s' % branch)
if output.endswith('nothing to commit, working directory clean'):
logging.info('Directory clean in branch: %s' % branch)
else:
raise Exception('Directory not clean, must commit:\n%s' % output)
logging.info('Switching to master branch')
git.checkout("master")
git.pull()
logging.info('Pulled latest changes from origin into master')
logging.info('Ensuring master has the latest changes')
output=git.pull()
if 'up-to-date' not in output:
raise Exception('Local copy was not up to date:\n%s' % output)
else:
logging.info('Local copy up to date')
logging.info('Switching back to branch: %s' % branch)
repo.git.checkout('remotes/origin/master')
def _git_checkout(self,tag,pkg=""):
import fs,git
fs.goto(os.path.join(fs.projects(),'gaudi',pkg))
lbranches,rbranches = git.branches()
if tag in lbranches:
git.checkout(tag)
else:
git.checkout('origin/'+tag,tag)
fs.goback()
return
def verify_service(team, branch, service_port, host_port, config_file):
config = load_config(config_file)
repo_owner = config['repo_owner']
repo_name = config['teams'][team]['repo_name']
container_name = "%s-%s" % (repo_name, branch)
clone(repo_owner, repo_name)
docker_cleanup(container_name)
checkout(repo_name, branch)
setup(repo_name, container_name, int(service_port), int(host_port))
check_liveness(container_name, int(host_port))
docker_cleanup(container_name)
rmdir(repo_name)
sys.exit()
def build(self, sha):
logging.info("Doing work for %s:%s", self.__repos, sha)
try:
source_path = os.path.join(config.build_root, self.__repos)
git.checkout(self.__repos, sha, source_path)
build.make(source_path)
self.__setStatus("success", sha)
except Exception as e:
logging.exception("Exception %s while doing work", e)
self.__setStatus("failure", sha)
github.addComment(self.__repos, sha,
"### TinyCI build failure:\n%s" % (e))
logging.info("Finished work for %s:%s", self.__repos, sha)
def test_show_stderr_already_on(self):
stdout, stderr = git.checkout(self.current_branch_name)
self.assertFalse(git.show_stderr(stderr, self.current_branch_name),
msg=(f"\nstderr :\n{stderr}\n"
"stdout :\n"
f"{stdout}"))
def process_repository(repository_full_path):
repo_result = {}
if os.path.isdir(repository_full_path
) and not ignore.is_ignore_path(repository_full_path):
repository = os.path.basename(repository_full_path)
# process repository
cwd_backup = os.getcwd()
os.chdir(repository_full_path)
msgo, msge = git.reset_hard_head()
if 'CONFLICT' in msgo or msge:
repo_result['reset'] = get_error_info('git reset --hard HEAD',
msgo, msge, repository)
msgo, msge = git.checkout('master')
if 'CONFLICT' in msgo or "Already on 'master'" != msge.strip():
repo_result['checkout'] = get_error_info('git checkout master',
msgo, msge, repository)
msgo, msge = git.pull()
if 'CONFLICT' in msgo or ('error' in msge) or ('fatal' in msge):
repo_result['pull'] = get_error_info('git pull', msgo, msge,
repository)
msgo, msge = git.status()
if 'CONFLICT' in msgo or msge:
repo_result['status'] = get_error_info('git status', msgo, msge,
repository)
os.chdir(cwd_backup)
return repo_result
def build_repo(repository, ref, docker_repo, docker_tag, namespace, push, registry):
docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo)
img_id = None
dst_folder = None
if '{0}@{1}'.format(repository, ref) not in processed.keys():
logger.info('Cloning {0} (ref: {1})'.format(repository, ref))
if repository not in processed:
rep, dst_folder = git.clone(repository, ref)
processed[repository] = rep
processed_folders.append(dst_folder)
else:
dst_folder = git.checkout(processed[repository], ref)
if not 'Dockerfile' in os.listdir(dst_folder):
raise RuntimeError('Dockerfile not found in cloned repository')
logger.info('Building using dockerfile...')
img_id, logs = client.build(path=dst_folder, quiet=True)
else:
img_id = processed['{0}@{1}'.format(repository, ref)]
logger.info('Committing to {0}:{1}'.format(docker_repo,
docker_tag or 'latest'))
client.tag(img_id, docker_repo, docker_tag)
if push:
logger.info('Pushing result to registry {0}'.format(
registry or "default"))
if registry is not None:
docker_repo = '{0}/{1}'.format(registry, docker_repo)
logger.info('Also tagging {0}'.format(docker_repo))
client.tag(img_id, docker_repo, docker_tag)
client.push(docker_repo)
return img_id
def test_create_commit_message():
expected_msg = '''\
Implement {}
name: {}
url: https://leetcode.com/problems/{}
difficulty: {}
time: {} ms
time-rank: {} %
time-complexity: {}
space: {} MB
space-rank: {} %
space-complexity: {}
Fixes #{}
Signed-off-by: {} <{}>
'''.format('foo-bar', 'foo-bar', 'foo-bar', 3, 0, 100, 'O(1)', 0, 100, 'O(1)',
42, problem.GIT_USER_NAME, problem.GIT_USER_EMAIL)
try:
git.branch('-D', 'issue/42/foo-bar')
except:
pass
p = problem.new_problem_args(
'foo-bar', 42, 3,
'class Solution {\npublic:\nint fooBar() {\nreturn 0;\n}\n};\n')
p._time = 0
p._time_rank = 100
p._time_complexity = 'O(1)'
p._space = 0
p._space_rank = 100
p._space_complexity = 'O(1)'
git.checkout(initial_branch)
git.branch('-D', 'issue/42/foo-bar')
os.remove('foo-bar.cpp')
os.remove('foo-bar-test.cpp')
actual_msg = p.create_commit_message()
assert (actual_msg == expected_msg)
def my_branch(repo, branchName=None):
""" Create a new branch.
" If branchName isn't given, create a new branch name in the
" format 'new_branch_{#}' where {#} is a number.
"""
branches = repo.branches
git = repo.git
i = 0
## Choose a new branch name for convenience.
newBranch = ''
if (branchName == None):
while ('new_branch_' + str(i)) in branches:
i=i+1
newBranch = 'new_branch_' + str(i)
else:
newBranch = branchName
git.checkout('head', b=newBranch)
repo.remote.push()
def build_repo(repository, ref, docker_repo, docker_tag, namespace, push,
registry, repos_folder, logger):
''' Builds one line of a library file.
repository: URL of the git repository that needs to be built
ref: Git reference (or commit ID) that needs to be built
docker_repo: Name of the docker repository where the image will
end up.
docker_tag: Tag for the image in the docker repository.
namespace: Namespace for the docker repository.
push: If the image should be pushed at the end of the build
registry: URL to private registry where image should be pushed
repos_folder: Directory where repositories should be cloned
logger: Logger instance
'''
dst_folder = None
img_id = None
commit_id = None
if repos_folder:
# Repositories are stored in a fixed location and can be reused
dst_folder = os.path.join(repos_folder, docker_repo + _random_suffix())
docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo)
if '{0}@{1}'.format(repository, ref) not in processed.keys():
# Not already built
rep = None
logger.info('Cloning {0} (ref: {1})'.format(repository, ref))
if repository not in processed: # Repository not cloned yet
rep, dst_folder = git.clone(repository, ref, dst_folder)
processed[repository] = rep
processed_folders.append(dst_folder)
else:
rep = processed[repository]
if ref in rep.refs:
# The ref already exists, we just need to checkout
dst_folder = git.checkout(rep, ref)
else: # ref is not present, try pulling it from the remote origin
rep, dst_folder = git.pull(repository, rep, ref)
if not 'Dockerfile' in os.listdir(dst_folder):
raise RuntimeError('Dockerfile not found in cloned repository')
commit_id = rep.head()
logger.info('Building using dockerfile...')
img_id, logs = client.build(path=dst_folder, quiet=True)
else:
logger.info('This ref has already been built, reusing image ID')
img_id = processed['{0}@{1}'.format(repository, ref)]
if ref.startswith('refs/'):
commit_id = processed[repository].ref(ref)
else:
commit_id = ref
logger.info('Committing to {0}:{1}'.format(docker_repo,
docker_tag or 'latest'))
client.tag(img_id, docker_repo, docker_tag)
if push:
logger.info('Pushing result to registry {0}'.format(
registry or "default"))
push_repo(img_id, docker_repo, registry=registry, logger=logger)
return img_id, commit_id
def clone(git_params,
repo_path,
logfile,
workDir=None,
clone_once=False,
**kwargs):
git = Git(git_params, repo_path, clone_once, logfile)
git.clone(**kwargs)
if git.repo_cloned:
git.checkout()
git.apply_patch(workDir)
git.log()
def build_repo(repository, ref, docker_repo, docker_tag, namespace, push,
registry, repos_folder, logger):
''' Builds one line of a library file.
repository: URL of the git repository that needs to be built
ref: Git reference (or commit ID) that needs to be built
docker_repo: Name of the docker repository where the image will
end up.
docker_tag: Tag for the image in the docker repository.
namespace: Namespace for the docker repository.
push: If the image should be pushed at the end of the build
registry: URL to private registry where image should be pushed
repos_folder: Directory where repositories should be cloned
logger: Logger instance
'''
dst_folder = None
img_id = None
commit_id = None
if repos_folder:
# Repositories are stored in a fixed location and can be reused
dst_folder = os.path.join(repos_folder, docker_repo + _random_suffix())
docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo)
if '{0}@{1}'.format(repository, ref) not in processed.keys():
# Not already built
rep = None
logger.info('Cloning {0} (ref: {1})'.format(repository, ref))
if repository not in processed: # Repository not cloned yet
rep, dst_folder = git.clone(repository, ref, dst_folder)
processed[repository] = rep
processed_folders.append(dst_folder)
else:
rep = processed[repository]
if ref in rep.refs:
# The ref already exists, we just need to checkout
dst_folder = git.checkout(rep, ref)
else: # ref is not present, try pulling it from the remote origin
rep, dst_folder = git.pull(repository, rep, ref)
if not 'Dockerfile' in os.listdir(dst_folder):
raise RuntimeError('Dockerfile not found in cloned repository')
commit_id = rep.head()
logger.info('Building using dockerfile...')
img_id, logs = client.build(path=dst_folder, quiet=True)
else:
logger.info('This ref has already been built, reusing image ID')
img_id = processed['{0}@{1}'.format(repository, ref)]
if ref.startswith('refs/'):
commit_id = processed[repository].ref(ref)
else:
commit_id = ref
logger.info('Committing to {0}:{1}'.format(docker_repo, docker_tag
or 'latest'))
client.tag(img_id, docker_repo, docker_tag)
if push:
logger.info('Pushing result to registry {0}'.format(registry
or "default"))
push_repo(img_id, docker_repo, registry=registry, logger=logger)
return img_id, commit_id
def repo_change_branch(repo_name,my_branch_name):
""" Checkout to the branch """
repo=Repo('/home/admin/Desktop/practice/GitPython-0.3.2.RC1/abc/'+repo_name)
git = repo.git
try:
#heads = repo.heads
#master = heads.reference
#master.commit
#master.rename("new_name")
git.checkout('HEAD', b=my_branch_name)
print "Repo changed to branch"+my_branch_name
except Exception,e:
print "Branch doesn't exist"
git.branch('HEAD', b=my_branch_name)
git.checkout('HEAD', b=my_branch_name)
print "Branch created and shited too"
def my_branch(repo, branchName=None):
""" Create a new branch.
" If branchName isn't given, create a new branch name in the
" format 'new_branch_{#}' where {#} is a number.
"""
branches = repo.branches
git = repo.git
i = 0
## Choose a new branch name for convenience.
newBranch = ''
if (branchName == None):
while ('new_branch_' + str(i)) in branches:
i = i + 1
newBranch = 'new_branch_' + str(i)
else:
newBranch = branchName
git.checkout('head', b=newBranch)
repo.remote.push()
def merge(repo_dir, gaia_url, branch_to, branch_from):
git.delete_gaia(repo_dir)
t = util.time_start()
if os.path.exists(repo_dir):
print "Updating Gaia"
git.update_gaia(repo_dir, gaia_url)
print "Updated Gaia in %0.2f seconds" % util.time_end(t)
else:
print "Creating Gaia"
git.create_gaia(repo_dir, gaia_url) # This is sadly broken
print "Created Gaia in %0.2f seconds" % util.time_end(t)
print "Merging %s into branch %s" % (branch_from, branch_to)
if not branch_to in git.branches(repo_dir):
print >> sys.stderr, "Asking to merge into a branch that doesn't exist (%s)" % branch_to
return None
if not branch_from in git.branches(repo_dir):
print >> sys.stderr, "Asking to merge from a branch that doesn't exist (%s)" % branch_from
return None
git.checkout(repo_dir, branch_to)
start_commit = git.get_rev(repo_dir)
git.merge(repo_dir, branch_from, strategy="recursive")
end_commit = git.get_rev(repo_dir)
print "Merge range is %s..%s" % (start_commit[:7], end_commit[:7])
print git.log(repo_dir, "%s..%s" % (start_commit, end_commit), pretty="oneline")
print "Dry Run push"
git.push(repo_dir, remote="origin", branches=[branch_to], dry_run=True)
info = git.push(repo_dir, remote="origin", branches=[branch_to])
print "Would be pusing to %s" % info["url"]
for branch in info["branches"].keys():
s, e = info["branches"][branch]
print " %s: %s..%s" % (branch, s, e)
if util.ask_yn("Push for realises?"):
info = git.push(repo_dir, remote="origin", branches=[branch_to], dry_run=False)
print "Pushed to %s" % info["url"]
for branch in info["branches"].keys():
s, e = info["branches"][branch]
print " %s: %s..%s" % (branch, s, e)
comment(repo_dir, branch_to, "%s..%s" % (start_commit, end_commit))
def get_git(scheme,url,target,overwrite,tag):
import git
if os.path.exists(target + '/.git'):
if not overwrite: return
else:
if len(scheme) == 1: giturl = url
else: giturl = url[4:]
git.clone(giturl,target)
fs.goto(target)
git.fetch()
out = git.branch()
for line in out.split('\n'):
if not line: continue
if line[0] != '*': continue
out = line.split()[1]
break
#print out,tag
if out != tag:
lbranches,rbranches = git.branches()
if tag in lbranches:
git.checkout(tag)
else:
# git 1.5 does not put remotes/ like 1.6 does
from exception import CommandFailure
try:
git.checkout('origin/'+tag,tag)
except CommandFailure:
git.checkout('remotes/origin/'+tag,tag)
git.pull()
fs.goback()
return
def save_dag(dag: DAG,
tasks: list,
commit_message=None,
create_pr=False) -> None:
def sort_task(a, b):
if b['task_id'] in a['upstream']:
return 1
elif a['task_id'] in b['upstream']:
return -1
return 0
dag_file = dag.full_filepath + '.new'
with open(dag_file, 'w') as fh:
dg = DagGenerator(dag)
for task in sorted(tasks, key=functools.cmp_to_key(sort_task)):
dg.add_task(task)
dag_file_string = f'{dg.dag_string}\n{dg.tasks_string}\n{dg.dependencies_string}'
fh.write(dg.import_string)
fh.write(dag_file_string)
dag_bag = DagBag()
temp_dag = dag_bag.process_file(dag_file)
if not temp_dag:
# os.unlink(dag_file)
return False
else:
os.rename(dag_file, dag.full_filepath)
if commit_message is not None:
os.chdir(os.path.dirname(__file__) + '/kirby')
date = datetime.now()
git.checkout(f'code-changes-{date:%Y%m%d-%H%M%s}', new=True)
print(git.status())
# print(git.diff())
git.commit(commit_message, add_all=True)
# git.push()
# if create_pr:
# git.create_pr('Changes from airflow: ' + commit_message)
# git.checkout('master')
return True
return False
def _clone_or_checkout(self, url, ref, dst_folder, rep):
if rep:
try:
# The ref already exists, we just need to checkout
dst_folder = git.checkout(rep, ref)
except git.GitException:
# ref is not present, try pulling it from the remote origin
rep, dst_folder = git.pull(url, rep, ref)
return rep, dst_folder
if dst_folder:
rmtree(dst_folder)
return git.clone(url, ref, dst_folder)
def _clone_or_checkout(self, url, ref, dst_folder, rep):
if rep:
try:
# The ref already exists, we just need to checkout
dst_folder = git.checkout(rep, ref)
except git.GitException:
# ref is not present, try pulling it from the remote origin
rep, dst_folder = git.pull(url, rep, ref)
return rep, dst_folder
if dst_folder:
rmtree(dst_folder)
return git.clone(url, ref, dst_folder)
def pull_path(repository_path, b_verbose=False):
"""
cd to repository_path
git pull
return to original path
"""
org_path = repo_path.cd(repository_path)
clean_repo_before_update(b_verbose=b_verbose, caller_name='pull_path')
git.checkout('master', b_verbose=False)
stdout, stderr = git.pull(b_verbose=False)
# if there was an error during pull
clean_repo_after_error(
stdout,
stderr,
'pull_path',
b_verbose=False,
)
os.chdir(org_path)
def test_checkout_branch():
try:
git.branch('-D', 'issue/42/foo-bar')
except:
pass
args = Problem.default_args()
args.issue = 42
args.name = 'foo-bar'
p = Problem(args)
p.checkout_branch()
actual_branch_name = '{}'.format(repo.active_branch)
expected_branch_name = 'issue/{}/{}'.format(args.issue, args.name)
git.checkout(initial_branch)
try:
git.branch('-D', expected_branch_name)
except:
pass
assert (actual_branch_name == expected_branch_name)
def process_section(self, k, section):
# github url to browse commit of a repository
print('{sec} {url}/tree/{commit}'.format(
sec=section, url=self.config['urls'][section], commit=self.get_commit(
section),
))
# clone or update the repository
repo = ret.clone_or_pull_repo_cd(k,
self.config['urls'][section],
self.folder,
b_update_repo='True' == self.config['operation']['folder'],
b_tag_after_update=False,
)
# abs path to the repository
repo_path = os.path.join(self.folder, repo['name'])
git.checkout(self.get_commit(section), repo_path=repo_path)
self.comments = collect_comments_recursively(repo_path, self.comments)
return self.comments
def fetch_and_reset(repository_path,
b_verbose=False,
revision='origin/master',
remote='origin'):
"""
cd to repository_path
git fetch
git reset origin/master
return to original path
"""
org_path = repo_path.cd(repository_path)
clean_repo_before_update(b_verbose=b_verbose,
caller_name='fetch_and_reset')
git.checkout('master', b_verbose=b_verbose)
stdout, stderr = git.fetch(remote)
clean_repo_after_error(
stdout,
stderr,
'fetch_and_reset__fetch',
b_verbose=b_verbose,
)
stdout, stderr = git.reset_hard_revision(revision)
clean_repo_after_error(
stdout,
stderr,
'fetch_and_reset__reset',
b_verbose=b_verbose,
)
os.chdir(org_path)
def cherry_pick_real(git, branches, frm, to):
git.checkout("master")
git.pull()
for br in branches:
print(f"switching to {br}...")
git.checkout(br)
print(f"pulling from remote repo...")
git.pull()
print(f"cherry picking {frm}..{to}...")
git.cherry_pick(f"{frm}..{to}")
print(f"pushing to remote repo...")
git.push()
print(f"switching to master...")
git.checkout("master")
def checkout(llvm_commit, clang_commit, compilerrt_commit):
cwd = os.path.abspath(os.getcwd())
if not os.path.exists('llvm'):
git.clone(llvm_git_repo, ['-n'])
os.chdir('llvm')
git.checkout(llvm_commit)
os.chdir(os.path.join(cwd, 'llvm', 'tools'))
if not os.path.exists('clang'):
git.clone(clang_git_repo, ['-n'])
os.chdir('clang')
git.checkout(clang_commit)
os.chdir(os.path.join(cwd, 'llvm', 'projects'))
if not os.path.exists('compiler-rt'):
git.clone(compilerrt_git_repo, ['-n'])
os.chdir('compiler-rt')
git.checkout(compilerrt_commit)
os.chdir(cwd)
for header in headers:
h.append(header.replace(":", ' = "', 1) + '"')
h_str = "\n\t\t".join(h)
print(h_str)
if not os.path.isdir(telegraf_dir):
repo = git.Repo.clone_from(telegraf_git, telegraf_dir)
else:
repo = git.Repo(telegraf_dir)
git = repo.git
if not os.path.isdir(telegraf_dir + "/" + monit_node):
os.mkdir(telegraf_dir + "/" + monit_node)
try:
git.checkout(monit_node)
except:
git.checkout(b=monit_node)
#git.checkout("HEAD", b=monit_node)
filename = monit_node + "/" + f['applicationsolution_name'] + "_" + \
f['url'].replace("://", "_").replace("/","_") + ".conf"
filepath = telegraf_dir + "/" + filename
print(filename)
with open(filepath, 'w') as fi:
fi.write("[[inputs.url_monitor]]\n" + \
"\tapp = " + '"' + f['applicationsolution_name'] + '"\n' + \
"\taddress = " + '"' + f['url'] + '"\n' + \
"\tresponse_timeout = \"" + response_timeout + '"\n' + \
"\tmethod = " + '"' + f['method'] + '"\n' + \
def test_create_problem(monkeypatch):
solution = \
'''class Solution {
public:
int fooBar(int z) {
}
};
'''
my_stdin = 'foo-bar\n42\n3\n' + solution
my_stdout = ''
try:
git.branch('-D', 'issue/42/foo-bar')
except:
pass
monkeypatch.setattr('sys.stdin', io.StringIO(my_stdin))
p = problem.new_problem()
assert (p._name == 'foo-bar')
assert (p._camel == 'FooBar')
assert (p._issue == 42)
assert (p._branch == 'issue/42/foo-bar')
assert (p._difficulty == Difficulty.HARD)
assert (p._problem_template + '\n' == solution)
io_fail = False
impl_cpp = p._name + '.cpp'
test_cpp = p._name + '-test.cpp'
if not os.path.exists(impl_cpp):
io_fail = True
if not os.path.exists(test_cpp):
io_fail = True
git.checkout(initial_branch)
git.branch('-D', p._branch)
if io_fail:
try:
os.remove(impl_cpp)
except:
pass
try:
os.remove(test_cpp)
except:
pass
assert (os.path.exists(impl_cpp))
assert (os.path.exists(test_cpp))
os.remove(impl_cpp)
os.remove(test_cpp)
# expected = ''
# expected += 'Enter the problem name: '
# expected += 'Enter the GitHub issue number: '
# expected += 'Enter the difficulty level (1=easy, 2=medium, 3=hard): '
# expected += 'Enter the Solution template: '
# this needs using capsys as a function, but it's already monkeypatch
# my_stdout = capsys.readouterr()
# assert(my_stdout == expected)
assert (p._time == -1)
assert (p._time_rank == -1)
assert (p._time_complexity == '')
assert (p._space == -1)
assert (p._space_rank == -1)
assert (p._space_complexity == '')
def update_repo_to_commit(hook_data):
commit_hash = hook_data['after']
git.pull(commit_hash)
branch = get_branch_from_push(hook_data)
git.checkout(branch)
return
if ask_user("Is it OK to crush the CI pipeline? (this will make it unusable for a while)"):
print("Yeah! let's *DO THIS* (where 'this' is piss off devs)")
delay = 10
else:
print("OK Slow and steady wins the race... or some such bullshit")
delay = 600
web_repo = git.Repo("/Users/ahonnecke/Code/repos/web-batch/")
git = web_repo.git
for remote in web_repo.remotes:
remote.fetch()
git.checkout('master')
git.reset('--hard', 'upstream/master')
os.chdir("/Users/ahonnecke/Code/repos/web-batch/server")
g = Github(os.environ['SERVICEDADTOKEN'])
branch = web_repo.active_branch
print(f'Active Branch: {branch.name}')
cache_warmer_filename = "/Users/ahonnecke/Code/repos/web-batch/.cache-warmer"
g = Github(os.environ['SERVICEDADTOKEN'])
try:
org = g.get_organization('digital-assets-data')
def create():
if sys.getdefaultencoding() != 'utf-8':
reload(sys)
sys.setdefaultencoding('utf-8')
path = os.path.dirname(os.path.realpath(__file__))
path = os.path.join(path,'config.yaml')
yamlContent,ind,bsi = load_yaml_guess_indent(open(path))
print ('\n')
print ('\n')
# 'LenzBusiness' 'LenzMember'
print('-----------------------------------')
print('工程列表:')
count = 0
for p in yamlContent['project_list']:
count += 1
print(str(count)+'.'+p['prefix'])
print('-----------------------------------')
repo_index = int(raw_input('请输入工程名称索引:'))
print('-----------------------------------')
repo_name = yamlContent['project_list'][repo_index - 1]['repo_name']
prefix = yamlContent['project_list'][repo_index - 1]['prefix']
pm_name = ''
task_name = ''
print ('\n')
print('-----------------------------------')
print('生成 feature/时间_任务名称')
print('例子 feature/20190516_时间打点')
print('-----------------------------------')
print('-----------------------------------')
print('pm列表：')
count = 0
for p in yamlContent['pm_list']:
count += 1
print(str(count)+'.'+p)
pm_index = int(raw_input('请输入PM名字索引:'))
pm_name = yamlContent['pm_list'][pm_index-1]
print('-----------------------------------')
print ('\n')
print('-----------------------------------')
while task_name == '':
task_name = raw_input('请输入任务名称(不要带空格)：')
print('-----------------------------------')
taskName = getGitBranchNameFromTaskName(task_name)
date_string = time.strftime("%Y%m%d",time.localtime())
just_test_branch = date_string + '_' + taskName #用作文件名
test_branch = 'feature/' + date_string + '_' + taskName
print ('\n')
print ('\n')
in_text = ''
test_options = ''
print('-----------------------------------')
print('项目测试项：---------一行一个---------')
print('相机优化 ')
print('主任务列表优化 ')
print('最后输入 q 回车 结束输入')
print('-----------------------------------')
print('请输入项目测试项:')
count = 0
while in_text != 'q':
count += 1
in_text = raw_input()
if in_text != 'q':
test_options += str(count) + '.' + in_text
test_options += '\n'
print('-----------------------------------')
print ('\n')
#git 打新分支 默认 feature/xxx
repo = Repo('~/' + repo_name)
master = repo.heads.master
currentBranch = repo.head.reference
if currentBranch != master:
master.checkout()
git = repo.git
git.checkout('master',b=test_branch)
print('切分支成功：')
print(test_branch)
#yaml文件更新
config = TemplateConfig()
config.readConfigFromTemplate()
config.git_branch = test_branch
config.git_project_name = repo_name
config.test_options = test_options
config.project_pm = pm_name
config.project_name = prefix + ' ' + task_name
yaml_name = just_test_branch+'_config.yaml'
path = os.path.dirname(os.path.realpath(__file__))
yamlPath = os.path.join(path,'configs/' + yaml_name)
if not os.path.isfile(yamlPath):
os.system("touch " + yamlPath)
path = os.path.dirname(os.path.realpath(__file__))
with io.open(path+'/configs/configs','a',encoding='utf-8') as f:
f.write(yaml_name)
f.write(u'\n')
config.save(yamlPath)
print('存储到本地配置成功：')
print(test_options)
print(f'Latest chrome version {chrome_version}')
ubuntu_tag = get_bionic_version()
# ubuntu_tag = get_ubuntu_version()
print(f'Latest version {ubuntu_tag}')
ts = int(time.time())
tag = f'{IMGBASENAME}-{ubuntu_tag}-{chrome_version}-{ts}'
remote_tag = f'{GITUSERNAME}/{tag}'
repo_instance = git.Repo(repo_path)
git = repo_instance.git
if not args.keep:
git.reset('--hard', SRCPATH)
git.checkout(SRCPATH)
os.chdir(repo_path)
call(['git', 'fetch', '--all'])
git.fetch('--all')
def update_base_dockerfile(dockerfile):
print('Updating the base dockerfile')
with open(dockerfile, 'r') as reader:
content = reader.read()
content_new = re.sub(
'FROM ubuntu:.* as chrome',
r'FROM ubuntu:' + str(ubuntu_tag) + ' as chrome',
content,
flags=re.M
#checkout a copy of the wiki branch
print "Checkout %s" % (self.vcs.get("branch"))
self.checkoutBranch(self.vcs.get("branch"),repo=self.shadow)
self.shadow.active_branch.pull()
#switch back to master
print "Checkout %s" % (self.vcs.get("shadow_branch"))
self.checkoutBranch(self.vcs.get("shadow_branch"),repo=self.shadow)
self.shadow.active_branch.pull()
git.merge(self.vcs.get("branch"),self.vcs.get("shadow_master"))
git.push()
def checkoutBranch(self,branch,**kwargs):
print "checkout branch: %s" % (branch)
if "repo" in kwargs:
git = kwargs["repo"].git
repo=kwargs['repo']
else:
git = self.repo.git
repo=self.repo
print "Repo.active branch=%s" %(repo.active_branch)
match=False
if repo.active_branch.name != branch:
try:
b = repo.branches[branch]
except IndexError, e:
git.branch("-f",branch, "origin/%s" % (branch))
finally:
git.checkout(branch)
docker_tag = "ahonnecke/dad-base"
base_dockerfile = "/Users/ahonnecke/Code/repos/web-batch/docker/Dockerfile.base"
chrome_version = get_chrome_driver_version()
print(f'Latest chrome version {chrome_version}')
bionic_tag = get_bionic_version()
print(f'Latest bionic version {bionic_tag}')
web_repo = git.Repo("/Users/ahonnecke/Code/repos/web-batch/")
git = web_repo.git
git.reset('--hard', 'upstream/master')
git.checkout('upstream/master')
call(['git', 'fetch', '--all'])
git.fetch('--all')
with open(base_dockerfile, 'r') as reader:
content = reader.read()
content_new = re.sub(
'FROM ubuntu:.*',
r'FROM ubuntu:' + str(bionic_tag),
content,
flags=re.M
)
with open(base_dockerfile, "w") as writer:
writer.write(content_new)
except git.exc.GitCommandError as e:
# Get the git error message from stderr and output the message without
# extraneous characters.
message = e.stderr.find("fatal:")
sys.exit(e.stderr[message:-2])
forward_branch = "forward/" + source_branch
git = repo.git
# We do not want to use any pre-existing branch.
try:
git.branch('-D', forward_branch)
except:
pass
git.checkout('HEAD', b=forward_branch)
# If using a local repository we want to fetch from the remote.
if options.path_to_repo:
print "Fetching from remote repository."
git.fetch()
# TODO: Add merging behavior, see:
# http://gitpython.readthedocs.io/en/stable/tutorial.html#using-git-directly
# TODO: Add continue option. This should be used after conflicts are resolved.
# This will commit and push the changes.
if options.continue_fp:
sys.exit("The continue option is not yet supported.")
# TODO: It would be nice to then view the diff here (or be provided with a
# link of the diff) and then enable one to open a pull request locally. PR
def checkout_iff_branch_exists(repo_name,my_branch_name):
""" Checkout to the branch """
repo=Repo(os.getcwd()+"/"+repo_name)
git = repo.git
git.checkout('HEAD', b=my_branch_name)
print "Repo changed to branch "+my_branch_name
def test_references_and_objects(self, rw_dir):
# [1-test_references_and_objects]
import git
repo = git.Repo.clone_from(self._small_repo_url(), os.path.join(rw_dir, 'repo'), branch='master')
heads = repo.heads
master = heads.master # lists can be accessed by name for convenience
master.commit # the commit pointed to by head called master
master.rename('new_name') # rename heads
master.rename('master')
# ![1-test_references_and_objects]
# [2-test_references_and_objects]
tags = repo.tags
tagref = tags[0]
tagref.tag # tags may have tag objects carrying additional information
tagref.commit # but they always point to commits
repo.delete_tag(tagref) # delete or
repo.create_tag("my_tag") # create tags using the repo for convenience
# ![2-test_references_and_objects]
# [3-test_references_and_objects]
head = repo.head # the head points to the active branch/ref
master = head.reference # retrieve the reference the head points to
master.commit # from here you use it as any other reference
# ![3-test_references_and_objects]
# [4-test_references_and_objects]
log = master.log()
log[0] # first (i.e. oldest) reflog entry
log[-1] # last (i.e. most recent) reflog entry
# ![4-test_references_and_objects]
# [5-test_references_and_objects]
new_branch = repo.create_head('new') # create a new one
new_branch.commit = 'HEAD~10' # set branch to another commit without changing index or working trees
repo.delete_head(new_branch) # delete an existing head - only works if it is not checked out
# ![5-test_references_and_objects]
# [6-test_references_and_objects]
new_tag = repo.create_tag('my_new_tag', message='my message')
# You cannot change the commit a tag points to. Tags need to be re-created
self.failUnlessRaises(AttributeError, setattr, new_tag, 'commit', repo.commit('HEAD~1'))
repo.delete_tag(new_tag)
# ![6-test_references_and_objects]
# [7-test_references_and_objects]
new_branch = repo.create_head('another-branch')
repo.head.reference = new_branch
# ![7-test_references_and_objects]
# [8-test_references_and_objects]
hc = repo.head.commit
hct = hc.tree
hc != hct
hc != repo.tags[0]
hc == repo.head.reference.commit
# ![8-test_references_and_objects]
# [9-test_references_and_objects]
assert hct.type == 'tree' # preset string type, being a class attribute
assert hct.size > 0 # size in bytes
assert len(hct.hexsha) == 40
assert len(hct.binsha) == 20
# ![9-test_references_and_objects]
# [10-test_references_and_objects]
assert hct.path == '' # root tree has no path
assert hct.trees[0].path != '' # the first contained item has one though
assert hct.mode == 0o40000 # trees have the mode of a linux directory
assert hct.blobs[0].mode == 0o100644 # blobs have a specific mode though comparable to a standard linux fs
# ![10-test_references_and_objects]
# [11-test_references_and_objects]
hct.blobs[0].data_stream.read() # stream object to read data from
hct.blobs[0].stream_data(open(os.path.join(rw_dir, 'blob_data'), 'wb')) # write data to given stream
# ![11-test_references_and_objects]
# [12-test_references_and_objects]
repo.commit('master')
repo.commit('v0.8.1')
repo.commit('HEAD~10')
# ![12-test_references_and_objects]
# [13-test_references_and_objects]
fifty_first_commits = list(repo.iter_commits('master', max_count=50))
assert len(fifty_first_commits) == 50
# this will return commits 21-30 from the commit list as traversed backwards master
ten_commits_past_twenty = list(repo.iter_commits('master', max_count=10, skip=20))
assert len(ten_commits_past_twenty) == 10
assert fifty_first_commits[20:30] == ten_commits_past_twenty
# ![13-test_references_and_objects]
# [14-test_references_and_objects]
headcommit = repo.head.commit
assert len(headcommit.hexsha) == 40
assert len(headcommit.parents) > 0
assert headcommit.tree.type == 'tree'
assert headcommit.author.name == 'Sebastian Thiel'
assert isinstance(headcommit.authored_date, int)
assert headcommit.committer.name == 'Sebastian Thiel'
assert isinstance(headcommit.committed_date, int)
assert headcommit.message != ''
# ![14-test_references_and_objects]
# [15-test_references_and_objects]
import time
time.asctime(time.gmtime(headcommit.committed_date))
time.strftime("%a, %d %b %Y %H:%M", time.gmtime(headcommit.committed_date))
# ![15-test_references_and_objects]
# [16-test_references_and_objects]
assert headcommit.parents[0].parents[0].parents[0] == repo.commit('master^^^')
# ![16-test_references_and_objects]
# [17-test_references_and_objects]
tree = repo.heads.master.commit.tree
assert len(tree.hexsha) == 40
# ![17-test_references_and_objects]
# [18-test_references_and_objects]
assert len(tree.trees) > 0 # trees are subdirectories
assert len(tree.blobs) > 0 # blobs are files
assert len(tree.blobs) + len(tree.trees) == len(tree)
# ![18-test_references_and_objects]
# [19-test_references_and_objects]
assert tree['smmap'] == tree / 'smmap' # access by index and by sub-path
for entry in tree: # intuitive iteration of tree members
print(entry)
blob = tree.trees[0].blobs[0] # let's get a blob in a sub-tree
assert blob.name
assert len(blob.path) < len(blob.abspath)
assert tree.trees[0].name + '/' + blob.name == blob.path # this is how the relative blob path is generated
assert tree[blob.path] == blob # you can use paths like 'dir/file' in tree[...]
# ![19-test_references_and_objects]
# [20-test_references_and_objects]
assert tree / 'smmap' == tree['smmap']
assert tree / blob.path == tree[blob.path]
# ![20-test_references_and_objects]
# [21-test_references_and_objects]
# This example shows the various types of allowed ref-specs
assert repo.tree() == repo.head.commit.tree
past = repo.commit('HEAD~5')
assert repo.tree(past) == repo.tree(past.hexsha)
assert repo.tree('v0.8.1').type == 'tree' # yes, you can provide any refspec - works everywhere
# ![21-test_references_and_objects]
# [22-test_references_and_objects]
assert len(tree) < len(list(tree.traverse()))
# ![22-test_references_and_objects]
# [23-test_references_and_objects]
index = repo.index
# The index contains all blobs in a flat list
assert len(list(index.iter_blobs())) == len([o for o in repo.head.commit.tree.traverse() if o.type == 'blob'])
# Access blob objects
for (path, stage), entry in index.entries.items():
pass
new_file_path = os.path.join(repo.working_tree_dir, 'new-file-name')
open(new_file_path, 'w').close()
index.add([new_file_path]) # add a new file to the index
index.remove(['LICENSE']) # remove an existing one
assert os.path.isfile(os.path.join(repo.working_tree_dir, 'LICENSE')) # working tree is untouched
assert index.commit("my commit message").type == 'commit' # commit changed index
repo.active_branch.commit = repo.commit('HEAD~1') # forget last commit
from git import Actor
author = Actor("An author", "*****@*****.**")
committer = Actor("A committer", "*****@*****.**")
# commit by commit message and author and committer
index.commit("my commit message", author=author, committer=committer)
# ![23-test_references_and_objects]
# [24-test_references_and_objects]
from git import IndexFile
# loads a tree into a temporary index, which exists just in memory
IndexFile.from_tree(repo, 'HEAD~1')
# merge two trees three-way into memory
merge_index = IndexFile.from_tree(repo, 'HEAD~10', 'HEAD', repo.merge_base('HEAD~10', 'HEAD'))
# and persist it
merge_index.write(os.path.join(rw_dir, 'merged_index'))
# ![24-test_references_and_objects]
# [25-test_references_and_objects]
empty_repo = git.Repo.init(os.path.join(rw_dir, 'empty'))
origin = empty_repo.create_remote('origin', repo.remotes.origin.url)
assert origin.exists()
assert origin == empty_repo.remotes.origin == empty_repo.remotes['origin']
origin.fetch() # assure we actually have data. fetch() returns useful information
# Setup a local tracking branch of a remote branch
empty_repo.create_head('master', origin.refs.master).set_tracking_branch(origin.refs.master)
origin.rename('new_origin') # rename remotes
# push and pull behaves similarly to `git push|pull`
origin.pull()
origin.push()
# assert not empty_repo.delete_remote(origin).exists() # create and delete remotes
# ![25-test_references_and_objects]
# [26-test_references_and_objects]
assert origin.url == repo.remotes.origin.url
cw = origin.config_writer
cw.set("pushurl", "other_url")
cw.release()
# Please note that in python 2, writing origin.config_writer.set(...) is totally safe.
# In py3 __del__ calls can be delayed, thus not writing changes in time.
# ![26-test_references_and_objects]
# [27-test_references_and_objects]
hcommit = repo.head.commit
hcommit.diff() # diff tree against index
hcommit.diff('HEAD~1') # diff tree against previous tree
hcommit.diff(None) # diff tree against working tree
index = repo.index
index.diff() # diff index against itself yielding empty diff
index.diff(None) # diff index against working copy
index.diff('HEAD') # diff index against current HEAD tree
# ![27-test_references_and_objects]
# [28-test_references_and_objects]
# Traverse added Diff objects only
for diff_added in hcommit.diff('HEAD~1').iter_change_type('A'):
print(diff_added)
# ![28-test_references_and_objects]
# [29-test_references_and_objects]
# Reset our working tree 10 commits into the past
past_branch = repo.create_head('past_branch', 'HEAD~10')
repo.head.reference = past_branch
assert not repo.head.is_detached
# reset the index and working tree to match the pointed-to commit
repo.head.reset(index=True, working_tree=True)
# To detach your head, you have to point to a commit directy
repo.head.reference = repo.commit('HEAD~5')
assert repo.head.is_detached
# now our head points 15 commits into the past, whereas the working tree
# and index are 10 commits in the past
# ![29-test_references_and_objects]
# [30-test_references_and_objects]
# checkout the branch using git-checkout. It will fail as the working tree appears dirty
self.failUnlessRaises(git.GitCommandError, repo.heads.master.checkout)
repo.heads.past_branch.checkout()
# ![30-test_references_and_objects]
# [31-test_references_and_objects]
git = repo.git
git.checkout('HEAD', b="my_new_branch") # create a new branch
git.branch('another-new-one')
git.branch('-D', 'another-new-one') # pass strings for full control over argument order
git.for_each_ref() # '-' becomes '_' when calling it
# ![31-test_references_and_objects]
# [32-test_references_and_objects]
ssh_executable = os.path.join(rw_dir, 'my_ssh_executable.sh')
with repo.git.custom_environment(GIT_SSH=ssh_executable):
# Note that we don't actually make the call here, as our test-setup doesn't permit it to
# succeed.
# It will in your case :)
repo.remotes.origin.fetch
def install(idp_fqdn=FQDN):
"""
Installe un serveur d'indentité disponible à *idp_fqdn*.
"""
# DEPS
lasso.install()
mysql.install()
ssl.install()
git.install()
apache.install()
apache.add_mod_rewrite()
apache.add_mod_ssl()
apache.add_mod_wsgi()
venv.install_virtualenv()
runcmd('apt-get install -y python-ldap')
runcmd('apt-get install -y python-mysqldb --force-yes')
# DB
db_name = idp_fqdn.replace('.', '_')
mysql.setup_db(db_name)
# WEB
ssl.create_certificats(idp_fqdn)
extra = {'venv': venv.get_path(idp_fqdn), }
apache.add_vhost(idp_fqdn, 'idp_vhost.txt', extra)
# SOURCE
git.clone('git://git.auf.org/authentic2', idp_fqdn)
git.checkout(idp_fqdn, 'master')
# VIRTUALENV
venv.mkenv(idp_fqdn)
bin_pip = venv.get_bin_pip(idp_fqdn)
with cd(git.home(idp_fqdn)):
git.sudo("%s install -r requirements.txt" % bin_pip)
git.sudo("%s install django-auth-ldap" % bin_pip)
# WSGI
data = {
'project_path': git.home(idp_fqdn),
'venv': venv.get_path(idp_fqdn),
}
filename = os.path.join(TPL_DIR, 'idp_wsgi.txt')
destination = os.path.join(venv.get_bin_path(idp_fqdn), 'idp_wsgi.py')
upload_template(
filename,
destination,
context=data,
use_sudo=True,)
runcmd('chown %s:%s %s' % (git.GIT_USER, git.GIT_GROUP, destination,))
runcmd('chmod 644 %s' % (destination, ))
# LOG file
log_file = os.path.join(git.home(idp_fqdn), 'log.log')
runcmd('touch %s' % log_file)
runcmd('chmod g+w %s' % log_file)
runcmd('chown %s:www-data %s' % (git.GIT_USER, log_file))
# CONF
data.update({
'db_name': db_name,
'db_user': db_name,
'db_password': db_name,
})
filename = os.path.join(TPL_DIR, 'idp_local_settings.txt')
destination = os.path.join(
git.home(idp_fqdn),
'aufcustom',
'local_settings.py')
upload_template(
filename,
destination,
context=data,
use_sudo=True,)
runcmd('chown %s:%s %s' % (git.GIT_USER, git.GIT_GROUP, destination,))
# manage.py
data.update({
'venv': venv.get_path(idp_fqdn),
})
filename = os.path.join(TPL_DIR, 'idp_manage.txt')
destination = os.path.join(
git.home(idp_fqdn),
'manage.py')
upload_template(
filename,
destination,
context=data,
use_sudo=True,)
runcmd('chown %s:%s %s' % (git.GIT_USER, git.GIT_GROUP, destination,))
runcmd('chmod +x %s' % (destination,))
git.sudo('%s syncdb --migrate --noinput' % (destination,))
git.sudo('%s collectstatic --noinput' % (destination,))
apache.restart()
