def _prepare_output_git(self, branch, input_work_dir, output_work_dir,
output_reference):
start_commit = self.output_branch_name(branch)
new_branch = False
try:
git.clone(
output_reference,
output_work_dir,
options=['--branch',
self.output_branch_name(branch), '-q'])
git.set_origin_url(self._output_tree, output_work_dir)
try:
git.commit_env_vars(self.output_branch_name(branch),
tree=output_work_dir)
except git.GitError:
shutil.rmtree(output_work_dir)
self.debug('clone incorrect - new branch?')
raise Exception('clone incorrect - new branch?')
except:
# assume error was due to branch not found
new_branch = True
start_commit = self._check_new_branch(branch, input_work_dir,
output_work_dir,
output_reference)
git.reset(['--hard', start_commit], tree=output_work_dir)
try:
git.fetch('refs/notes/mirror:refs/notes/mirror',
tree=output_work_dir)
except:
pass
return start_commit, new_branch
def benchmark(git_repo,
src_dir,
branches,
git_origin,
images,
qmake_file,
binary,
clone_url=None):
if (not os.path.exists(git_repo)):
if (clone_url != None):
git.clone_repo(clone_url, git_repo)
git.fetch(git_repo)
git.replace_remote(git_repo, clone_url)
else:
git.clone_repo(git_origin, git_repo)
git.fetch_all(git_repo)
results = {"computer_name": computer_name()}
for branch in branches:
git.checkout_branch(git_repo, branch)
git.clean(src_dir)
compile.compile(qmake_file, cwd=src_dir)
r = {"revision": git.current_revision(git_repo)}
r["results"] = run(src_dir + "/" + binary, images)
results[branch] = r
return results
def get_git(scheme,url,target,overwrite,tag):
import git
if os.path.exists(target + '/.git'):
if not overwrite: return
else:
if len(scheme) == 1: giturl = url
else: giturl = url[4:]
git.clone(giturl,target)
fs.goto(target)
git.fetch()
out = git.branch()
for line in out.split('\n'):
if not line: continue
if line[0] != '*': continue
out = line.split()[1]
break
#print out,tag
if out != tag:
lbranches,rbranches = git.branches()
if tag in lbranches:
git.checkout(tag)
else:
# git 1.5 does not put remotes/ like 1.6 does
from exception import CommandFailure
try:
git.checkout('origin/'+tag,tag)
except CommandFailure:
git.checkout('remotes/origin/'+tag,tag)
git.pull()
fs.goback()
return
def get_branch_files(repository, check_upstream=False):
if check_upstream and repository.github.upstream is not None:
git.fetch(
'upstream', '--no-tags', '%s:refs/remotes/upstream/%s' % \
(repository.github.branch, repository.github.branch))
diff_output = git.diff(
'--name-only', 'origin/%s..upstream/%s' % \
(repository.github.branch, repository.github.branch),
strip=False)
git.rebase('upstream/%s' % repository.github.branch)
else:
diff_output = '\n'.join([
line[3:] for line in git.status('status', '-s', strip=False)
if len(line) > 3
])
new_files = get_eligible_files(repository, diff_output, 'en')
lstree_output = git.ls_tree('-r',
'--name-only',
repository.github.branch,
strip=False)
all_files = get_eligible_files(repository, lstree_output, 'en')
return new_files, all_files
def git_update_remote_heads():
git.fetch()
remote_branch_heads = git.get_all_remote_branch_heads(branch_regexp=config.branch_name_regexp)
dbcon.executemany(
"insert or ignore into branch(branch, git_remote_head_remote) values (?, ?);", remote_branch_heads
)
dbcon.executemany(
"update branch set git_remote_head_remote=? where branch=?;",
map(lambda (branch, head): (head, branch), remote_branch_heads),
)
git_master_head_remote = git.get_head("remotes/origin/master")
dbcon.execute("""update branch set git_master_head_remote=?;""", (git_master_head_remote,))
def benchmark(git_repo, src_dir, branches, git_origin, images, qmake_file, binary, clone_url=None):
if (not os.path.exists(git_repo)):
if (clone_url != None):
git.clone_repo(clone_url, git_repo)
git.fetch(git_repo)
git.replace_remote(git_repo, clone_url)
else:
git.clone_repo(git_origin, git_repo)
git.fetch_all(git_repo)
results = {"computer_name": computer_name()}
for branch in branches:
git.checkout_branch(git_repo, branch)
git.clean(src_dir)
compile.compile(qmake_file,cwd=src_dir)
r = {"revision": git.current_revision(git_repo)}
r["results"] = run(src_dir+"/"+binary, images)
results[branch] = r
return results
def __call__(self):
self.log.write('New changesets have been pushed.\n')
# See if the push changed something in the master branch
git.reset('origin/master')
current_id = git.id()
git.fetch()
git.reset(self.revision)
if current_id != self.revision:
self.log.write('The master branch has new commits.\n')
# Make a new development snapshot
builder = snapshotbuilder.SnapshotBuilder(self.amqp_connection, self.revision, self.log, 'openclonk', False)
# TODO: Remove all other snapshot builders from the queue
self.queue.put(50, builder)
# Also make a new mape build. In principle we could do this only if something in the
# mape directory or any of the other files used by mape change, but let's keep it simple here.
builder = snapshotbuilder.SnapshotBuilder(self.amqp_connection, self.revision, self.log, 'mape', False)
# TODO: Remove all other snapshot builders from the queue
self.queue.put(70, builder)
# See if something in the docs directory has changed
log = git.log('docs', current_id, self.revision, 'oneline')
if len(log) > 1 or (len(log) == 1 and log[0] != current_id):
# TODO: Remove all other doc builders from the queue
builder = docbuilder.DocBuilder(self.revision, self.log)
self.queue.put(80, builder)
else:
self.log.write('The master branch has no new commits.\n')
# TODO: Make a release if the version changed
return True
def fetch_and_reset(repository_path,
b_verbose=False,
revision='origin/master',
remote='origin'):
"""
cd to repository_path
git fetch
git reset origin/master
return to original path
"""
org_path = repo_path.cd(repository_path)
clean_repo_before_update(b_verbose=b_verbose,
caller_name='fetch_and_reset')
git.checkout('master', b_verbose=b_verbose)
stdout, stderr = git.fetch(remote)
clean_repo_after_error(
stdout,
stderr,
'fetch_and_reset__fetch',
b_verbose=b_verbose,
)
stdout, stderr = git.reset_hard_revision(revision)
clean_repo_after_error(
stdout,
stderr,
'fetch_and_reset__reset',
b_verbose=b_verbose,
)
os.chdir(org_path)
def _config(self, remote, conf, groups):
"""
Builds the groups file and project.config file for a project.
@param remote - gerrit.Remote object
@param conf - Dict containing git config information
@param groups - List of groups
"""
if not self.config:
return
msg = "Project %s: Configuring." % self.name
logger.info(msg)
print msg
repo_dir = '~/tmp'
repo_dir = os.path.expanduser(repo_dir)
repo_dir = os.path.abspath(repo_dir)
uuid_dir = str(uuid4())
repo_dir = os.path.join(repo_dir, uuid_dir)
# Make Empty directory - We want this to stop and fail on OSError
logger.debug(
"Project %s: Creating directory %s" % (self.name, repo_dir)
)
os.makedirs(repo_dir)
# Save the current working directory
old_cwd = os.getcwd()
origin = 'origin'
try:
# Change cwd to that repo
os.chdir(repo_dir)
# Git init empty directory
git.init()
# Add remote origin
ssh_url = 'ssh://%s@%s:%s/%s' % (
remote.username,
remote.host,
remote.port,
self.name
)
git.add_remote(origin, ssh_url)
# Fetch refs/meta/config for project
refspec = 'refs/meta/config:refs/remotes/origin/meta/config'
git.fetch(origin, refspec)
# Checkout refs/meta/config
git.checkout_branch('meta/config')
# Get md5 of existing config
_file = os.path.join(repo_dir, 'project.config')
contents = ''
try:
with open(_file, 'r') as f:
contents = f.read()
except IOError:
pass
existing_md5 = hashlib.md5(contents).hexdigest()
# Get md5 of new config
with open(self.config, 'r') as f:
contents = f.read()
new_md5 = hashlib.md5(contents).hexdigest()
msg = "Project %s: Md5 comparision\n%s\n%s"
msg = msg % (self.name, existing_md5, new_md5)
logger.debug(msg)
print msg
# Only alter if checksums do not match
if existing_md5 != new_md5:
logger.debug(
"Project %s: config md5's are different." % self.name
)
# Update project.config file
_file = os.path.join(repo_dir, 'project.config')
with open(_file, 'w') as f:
f.write(contents)
# Update groups file
group_contents = groups_file_contents(groups)
_file = os.path.join(repo_dir, 'groups')
with open(_file, 'w') as f:
f.write(group_contents)
# Git config user.email
git.set_config('user.email', conf['git-config']['email'])
# Git config user.name
git.set_config('user.name', conf['git-config']['name'])
# Add groups and project.config
git.add(['groups', 'project.config'])
# Git commit
git.commit(message='Setting up %s' % self.name)
# Git push
git.push(origin, refspecs='meta/config:refs/meta/config')
logger.info("Project %s: pushed configuration." % self.name)
else:
msg = "Project %s: config unchanged." % self.name
logger.info(msg)
print msg
finally:
# Change to old current working directory
os.chdir(old_cwd)
# Attempt to clean up created directory
shutil.rmtree(repo_dir)
base_dockerfile = "/Users/ahonnecke/Code/repos/web-batch/docker/Dockerfile.base"
chrome_version = get_chrome_driver_version()
print(f'Latest chrome version {chrome_version}')
bionic_tag = get_bionic_version()
print(f'Latest bionic version {bionic_tag}')
web_repo = git.Repo("/Users/ahonnecke/Code/repos/web-batch/")
git = web_repo.git
git.reset('--hard', 'upstream/master')
git.checkout('upstream/master')
call(['git', 'fetch', '--all'])
git.fetch('--all')
with open(base_dockerfile, 'r') as reader:
content = reader.read()
content_new = re.sub(
'FROM ubuntu:.*',
r'FROM ubuntu:' + str(bionic_tag),
content,
flags=re.M
)
with open(base_dockerfile, "w") as writer:
writer.write(content_new)
writer.close()
with open(base_dockerfile, 'r') as reader:
forward_branch = "forward/" + source_branch
git = repo.git
# We do not want to use any pre-existing branch.
try:
git.branch('-D', forward_branch)
except:
pass
git.checkout('HEAD', b=forward_branch)
# If using a local repository we want to fetch from the remote.
if options.path_to_repo:
print "Fetching from remote repository."
git.fetch()
# TODO: Add merging behavior, see:
# http://gitpython.readthedocs.io/en/stable/tutorial.html#using-git-directly
# TODO: Add continue option. This should be used after conflicts are resolved.
# This will commit and push the changes.
if options.continue_fp:
sys.exit("The continue option is not yet supported.")
# TODO: It would be nice to then view the diff here (or be provided with a
# link of the diff) and then enable one to open a pull request locally. PR
# should be named something like 'Fwdport forward/rel/6.2.5 to 6.3'. It
# should be opened against the target branch.
# TODO: Cleanup any cloned repository.
def _config(self, remote, conf, groups):
"""
Builds the groups file and project.config file for a project.
@param remote - gerrit.Remote object
@param conf - Dict containing git config information
@param groups - List of groups
"""
if not self.config:
return
msg = "Project %s: Configuring." % self.name
logger.info(msg)
print msg
repo_dir = '~/tmp'
repo_dir = os.path.expanduser(repo_dir)
repo_dir = os.path.abspath(repo_dir)
uuid_dir = str(uuid4())
repo_dir = os.path.join(repo_dir, uuid_dir)
# Make Empty directory - We want this to stop and fail on OSError
logger.debug("Project %s: Creating directory %s" %
(self.name, repo_dir))
os.makedirs(repo_dir)
# Save the current working directory
old_cwd = os.getcwd()
origin = 'origin'
try:
# Change cwd to that repo
os.chdir(repo_dir)
# Git init empty directory
git.init()
# Add remote origin
ssh_url = 'ssh://%s@%s:%s/%s' % (remote.username, remote.host,
remote.port, self.name)
git.add_remote(origin, ssh_url)
# Fetch refs/meta/config for project
refspec = 'refs/meta/config:refs/remotes/origin/meta/config'
git.fetch(origin, refspec)
# Checkout refs/meta/config
git.checkout_branch('meta/config')
# Get md5 of existing config
_file = os.path.join(repo_dir, 'project.config')
contents = ''
try:
with open(_file, 'r') as f:
contents = f.read()
except IOError:
pass
existing_md5 = hashlib.md5(contents).hexdigest()
# Get md5 of new config
with open(self.config, 'r') as f:
contents = f.read()
new_md5 = hashlib.md5(contents).hexdigest()
msg = "Project %s: Md5 comparision\n%s\n%s"
msg = msg % (self.name, existing_md5, new_md5)
logger.debug(msg)
print msg
# Only alter if checksums do not match
if existing_md5 != new_md5:
logger.debug("Project %s: config md5's are different." %
self.name)
# Update project.config file
_file = os.path.join(repo_dir, 'project.config')
with open(_file, 'w') as f:
f.write(contents)
# Update groups file
group_contents = groups_file_contents(groups)
_file = os.path.join(repo_dir, 'groups')
with open(_file, 'w') as f:
f.write(group_contents)
# Git config user.email
git.set_config('user.email', conf['git-config']['email'])
# Git config user.name
git.set_config('user.name', conf['git-config']['name'])
# Add groups and project.config
git.add(['groups', 'project.config'])
# Git commit
git.commit(message='Setting up %s' % self.name)
# Git push
git.push(origin, refspecs='meta/config:refs/meta/config')
logger.info("Project %s: pushed configuration." % self.name)
else:
msg = "Project %s: config unchanged." % self.name
logger.info(msg)
print msg
finally:
# Change to old current working directory
os.chdir(old_cwd)
# Attempt to clean up created directory
shutil.rmtree(repo_dir)
os.unlink(ssh_wrapper_path)
remote = f'[email protected]:{os.environ["GITHUB_REPOSITORY"]}'
else:
@contextmanager
def git_auth():
"""
No-op context manager.
"""
yield
remote = f'https://{token}@github.com/{os.environ["GITHUB_REPOSITORY"]}'
git.remote('add', 'gh-token', remote)
with git_auth():
git.fetch('gh-token')
# reset will fail if 'generated-documentation` branch doesn't yet exist.
# That's fine, it will exist after our push. Just not the error and move on.
try:
git.reset('gh-token/generated-documentation')
except GitCommandError:
print(NOTICE + "Couldn't git reset generated-documentation." + ENDC)
print(NOTICE +
"This error is expected if the branch doesn't exist yet." + ENDC)
print(INFO + "Running 'mkdocs gh-deploy'." + ENDC)
os.chdir(docs_build_dir)
rslt = os.system(
'mkdocs gh-deploy --verbose --clean --remote-name gh-token '
'--remote-branch generated-documentation')
if rslt != 0:
def __call__(self):
if self.dry_release:
self.log.write('Dry-Releasing revision %s...\n' % self.revision)
else:
self.log.write('Releasing revision %s...\n' % self.revision)
# Update to revision and get hexadecimal ID.
# TODO: Reset back to 'origin/master' afterwards
# TODO: The following could be checked easier maybe...
prefix = ''
if not self.revision.startswith('v'):
for x in 'ghijklmnopqrstuvwxyz':
if len(prefix) == 0 and x in self.revision:
prefix = 'origin/'
git.fetch()
git.reset('%s%s' % (prefix, self.revision))
revision = git.id()[:12]
(major, minor) = self.parse_version_file('Version.txt')
self.log.write('==> Version %d.%d\n' % (major, minor))
dry_suffix = ''
if self.dry_release: dry_suffix = '-dry'
archive = os.path.join(self.archive_dir, '%d.%d%s' % (major, minor, dry_suffix))
if os.path.exists(archive):
self.log.write('Archive directory %s exists already. Clearing...\n' % archive)
shutil.rmtree(archive)
os.mkdir(archive)
# Copy game content to archive
self.log.write('Copying and packing game content to archive...\n')
content = [] # game content
others = [] # other misc. non-architecture dependent files
for filename, stream in contentiter.ContentIter(groupcontent.release):
self.log.write('%s...\n' % filename)
if contentiter.ContentIter.is_group_file(filename):
content.append(filename)
else:
others.append(filename)
destination = os.path.join(archive, filename)
_create_and_open(destination).write(stream.read()) # TODO: copyfileobj?
# Create architecture specific files:
all_files = {}
for arch in arches.arches:
# Obtain clonk and c4group binaries and dependencies and save in archive/$arch
self.log.write('Creating architecture dependent files for %s...\n' % arch)
archdir = os.path.join(archive, arch)
os.mkdir(archdir)
binaries = []
if arch.startswith('darwin'):
result, uuid = autobuild.obtain(self.amqp_connection, revision, arch, ['openclonk'])
filename, stream = result[0]
_create_and_open(os.path.join(archdir, filename)).write(stream.read())
binaries.append(filename)
else:
# Copy both binaries and dependencies into archive.
for filename, stream in architer.ArchIter(self.amqp_connection, arch, revision, 'openclonk'):
_create_and_open(os.path.join(archdir, filename)).write(stream.read())
if architer.ArchIter.is_executable(filename):
os.chmod(os.path.join(archdir, filename), 0755)
binaries.append(filename)
# Create distribution directory and copy both common and
# architecture dependent files there.
distdir = os.path.join(archdir, 'openclonk-%d.%d' % (major, minor))
os.mkdir(distdir)
if not arch.startswith('darwin'):
for filename in content + others:
_ensure_dir_exists(os.path.dirname(os.path.join(distdir, filename)))
shutil.copy(os.path.join(archive, filename), os.path.join(distdir, filename))
for filename in binaries:
_ensure_dir_exists(os.path.dirname(os.path.join(distdir, filename)))
shutil.copy(os.path.join(archdir, filename), os.path.join(distdir, filename))
# Create full installation package
file = self.pack_full_installation(distdir, arch)
all_files[arch] = file
# TODO: Create a source tarball
uploader = upload.Uploader(self.log, self.dry_release)
# TODO uncomment when source tarball created
#uploader.release_file(source_package_filename, (major, minor))
for arch,file in all_files.items():
uploader.release_binaries(file, arch, (major, minor))
os.unlink(file)
# Remove the archive if this was a dry release
if self.dry_release:
shutil.rmtree(archive)
return True
