def upconvert_bloom_to_config_branch():
global _has_checked_bloom_branch
if _has_checked_bloom_branch:
return
# Assert that this repository does not have multiple remotes
check_for_multiple_remotes()
if get_root() is None:
# Not a git repository
return
track_branches(['bloom', BLOOM_CONFIG_BRANCH])
if show('bloom', PLACEHOLDER_FILE) is not None:
return
if show('bloom', 'bloom.conf') is not None:
# Wait for the bloom.conf upconvert...
return
if not branch_exists('bloom'):
return
_has_checked_bloom_branch = True
info("Moving configurations from deprecated 'bloom' branch "
"to the '{0}' branch.".format(BLOOM_CONFIG_BRANCH))
tmp_dir = mkdtemp()
git_root = get_root()
try:
# Copy the new upstream source into the temporary directory
with inbranch('bloom'):
ignores = ('.git', '.gitignore', '.svn', '.hgignore', '.hg', 'CVS')
configs = os.path.join(tmp_dir, 'configs')
my_copytree(git_root, configs, ignores)
if [x for x in os.listdir(os.getcwd()) if x not in ignores]:
execute_command('git rm -rf ./*')
with open(PLACEHOLDER_FILE, 'w') as f:
f.write("""\
This branch ('bloom') has been deprecated in favor of storing settings and overlay files in the master branch.
Please goto the master branch for anything which referenced the bloom branch.
You can delete this branch at your convenience.
""")
execute_command('git add ' + PLACEHOLDER_FILE)
if has_changes():
execute_command('git commit -m "DEPRECATING BRANCH"')
if not branch_exists(BLOOM_CONFIG_BRANCH):
info("Creating '{0}' branch.".format(BLOOM_CONFIG_BRANCH))
create_branch(BLOOM_CONFIG_BRANCH, orphaned=True)
with inbranch(BLOOM_CONFIG_BRANCH):
my_copytree(configs, git_root)
execute_command('git add ./*')
if has_changes():
execute_command('git commit -m '
'"Moving configs from bloom branch"')
finally:
# Clean up
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
def upconvert_bloom_to_config_branch():
global _has_checked_bloom_branch
if _has_checked_bloom_branch:
return
# Assert that this repository does not have multiple remotes
check_for_multiple_remotes()
if get_root() is None:
# Not a git repository
return
track_branches(['bloom', BLOOM_CONFIG_BRANCH])
if show('bloom', PLACEHOLDER_FILE) is not None:
return
if show('bloom', 'bloom.conf') is not None:
# Wait for the bloom.conf upconvert...
return
if not branch_exists('bloom'):
return
_has_checked_bloom_branch = True
info("Moving configurations from deprecated 'bloom' branch "
"to the '{0}' branch.".format(BLOOM_CONFIG_BRANCH))
tmp_dir = mkdtemp()
git_root = get_root()
try:
# Copy the new upstream source into the temporary directory
with inbranch('bloom'):
ignores = ('.git', '.gitignore', '.svn', '.hgignore', '.hg', 'CVS')
configs = os.path.join(tmp_dir, 'configs')
my_copytree(git_root, configs, ignores)
if [x for x in os.listdir(os.getcwd()) if x not in ignores]:
execute_command('git rm -rf ./*')
with open(PLACEHOLDER_FILE, 'w') as f:
f.write("""\
This branch ('bloom') has been deprecated in favor of storing settings and overlay files in the master branch.
Please goto the master branch for anything which referenced the bloom branch.
You can delete this branch at your convenience.
""")
execute_command('git add ' + PLACEHOLDER_FILE)
if has_changes():
execute_command('git commit -m "DEPRECATING BRANCH"')
if not branch_exists(BLOOM_CONFIG_BRANCH):
info("Creating '{0}' branch.".format(BLOOM_CONFIG_BRANCH))
create_branch(BLOOM_CONFIG_BRANCH, orphaned=True)
with inbranch(BLOOM_CONFIG_BRANCH):
my_copytree(configs, git_root)
execute_command('git add ./*')
if has_changes():
execute_command('git commit -m '
'"Moving configs from bloom branch"')
finally:
# Clean up
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
def post_patch(self, destination, color='bluef'):
# Tag after patches have been applied
with inbranch(destination):
# Tag
tag_name = self.tag_names[destination]
if tag_exists(tag_name):
if self.interactive:
warning("Tag exists: " + tag_name)
warning("Do you wish to overwrite it?")
if not maybe_continue('y'):
error("Answered no to continue, aborting.")
return code.ANSWERED_NO_TO_CONTINUE
else:
warning("Overwriting tag: " + tag_name)
else:
info("Creating tag: " + tag_name)
execute_command('git tag -f ' + tag_name)
# Report of success
name = destination.split('/')[-1]
stackage, kind = self.packages[name]
distro = destination.split('/')[-2]
info(ansi(color) + "####" + ansi('reset'), use_prefix=False)
info(
ansi(color) + "#### " + ansi('greenf') + "Successfully" + \
ansi(color) + " generated '" + ansi('boldon') + distro + \
ansi('boldoff') + "' debian for " + kind + \
" '" + ansi('boldon') + stackage.name + ansi('boldoff') + "'" + \
" at version '" + ansi('boldon') + stackage.version + \
"-" + str(self.debian_inc) + ansi('boldoff') + "'" + \
ansi('reset'),
use_prefix=False
)
info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False)
def post_rebase(self, destination):
name = destination.split('/')[-1]
# Retrieve the package
package = self.packages[name]
# Handle differently if this is an arch vs distro branch
if destination in self.arch_branches:
info("Placing Arch template files into '{0}' branch."
.format(destination))
# Then this is an arch branch
# Place the raw template files
self.place_template_files()
else:
# This is a distro specific arch branch
# Determine the current package being generated
distro = destination.split('/')[-2]
# Create Arch packages for each distro
with inbranch(destination):
# To fit Arch Linux philosophy a bit better, we move all the source files into a subdirectory.
# Arch Linux doesn't support source distribution through a subdirectory; therefore we should ideally compress the sources or provide a URL.
# At this point in the generator, it is tricky to get the release URL. Furthermore it wouldn't fit bloom's patch mechanism very well.
# To work around, we copy the sources to the $srcdir at the beginning inside the prepare() function.
temp_dir = mkdtemp(dir='.')
for item in os.listdir("."):
itemsrc = os.path.abspath(item)
if os.path.basename(itemsrc) in ['.', '..', '.git', '.svn', 'arch',os.path.basename(temp_dir)]:
continue
itemdst = os.path.abspath(os.path.join(temp_dir,item))
execute_command('git mv ' + itemsrc + ' ' + itemdst)
execute_command('git mv ' + temp_dir + ' ' + name)
execute_command('git commit --amend --no-edit')
# Then generate the PKGBUILD
data = self.generate_arch(package, distro)
# And finally move the PKGBUILD to the root directory of the package.
for item in os.listdir("arch"):
itemsrc = os.path.abspath(os.path.join("arch",item))
if os.path.basename(itemsrc) in ['.', '..', '.git', '.svn']:
continue
itemdst = os.path.abspath(item)
execute_command('git mv ' + itemsrc + ' ' + itemdst)
execute_command('git commit --amend --no-edit')
# Create the tag name for later
self.tag_names[destination] = self.generate_tag_name(data)
# Update the patch configs
patches_branch = 'patches/' + destination
config = get_patch_config(patches_branch)
# Store it
self.store_original_config(config, patches_branch)
# Modify the base so import/export patch works
current_branch = get_current_branch()
if current_branch is None:
error("Could not determine current branch.", exit=True)
config['base'] = get_commit_hash(current_branch)
# Set it
set_patch_config(patches_branch, config)
def get_changelog_summary(release_tag):
summary = ""
for package in get_packages().values():
release_branch = '/'.join(release_tag.split('/')[:-1]).format(package=package.name)
if not branch_exists(release_branch):
continue
with inbranch(release_branch):
changelog = get_changelog_from_path(os.getcwd())
if changelog is None:
continue
for version, date, changes in changelog.foreach_version():
if version == package.version:
msgs = []
for change in changes:
msgs.extend([i for i in to_unicode(change).splitlines()])
msg = '\n'.join(msgs)
summary += """
## {package.name}
""".format(**locals())
if msg:
summary += """
```
{msg}
```
""".format(**locals())
else:
summary += """
- No changes
"""
return summary
def commit_summary():
global _summary_file
if get_root() is None:
return
if _summary_file is None:
return
if not os.path.exists(_summary_file.name):
return
try:
with inbranch('master'):
readme_name = 'README.md'
readme = ''
if os.path.isfile(readme_name):
with open(readme_name, 'r') as f:
readme = f.read()
_summary_file.close()
with open(_summary_file.name, 'r') as f:
readme = f.read() + "\n\n" + readme
with open(readme_name, 'w') as f:
f.write(readme)
execute_command('git add ' + readme_name)
if has_changes():
execute_command('git commit -m "Updating README.md"')
finally:
if _summary_file is not None:
_summary_file.close()
if os.path.exists(_summary_file.name):
os.remove(_summary_file.name)
def post_rebase(self, destination):
name = destination.split('/')[-1]
# Retrieve the package
package = self.packages[name]
# Handle differently if this is an rpm vs distro branch
if destination in self.rpm_branches:
info("Placing RPM template files into '{0}' branch."
.format(destination))
# Then this is an rpm branch
# Place the raw template files
self.place_template_files()
else:
# This is a distro specific rpm branch
# Determine the current package being generated
distro = destination.split('/')[-2]
# Create RPMs for each distro
with inbranch(destination):
data = self.generate_rpm(package, distro)
# Create the tag name for later
self.tag_names[destination] = self.generate_tag_name(data)
# Update the patch configs
patches_branch = 'patches/' + destination
config = get_patch_config(patches_branch)
# Store it
self.store_original_config(config, patches_branch)
# Modify the base so import/export patch works
current_branch = get_current_branch()
if current_branch is None:
error("Could not determine current branch.", exit=True)
config['base'] = get_commit_hash(current_branch)
# Set it
set_patch_config(patches_branch, config)
def post_rebase(self, destination):
# Determine the current package being generated
name = destination.split('/')[-1]
distro = destination.split('/')[-2]
# Retrieve the stackage
stackage, kind = self.packages[name]
# Ask to continue if interactive
if self.interactive:
if not maybe_continue('y'):
error("Answered no to continue, aborting.")
return code.ANSWERED_NO_TO_CONTINUE
### Start debian generation
# Get time of day
from dateutil import tz
stamp = datetime.datetime.now(tz.tzlocal())
# Convert stackage to debian data
data = self.convert_stackage_to_debian_data(stackage, kind)
# Get apt_installer from rosdep
from rosdep2.catkin_support import get_installer
self.apt_installer = get_installer(APT_INSTALLER)
# Create debians for each distro
with inbranch(destination):
self.generate_debian(data, stamp, distro)
# Create the tag name for later
self.tag_names[destination] = self.generate_tag_name(data)
# Update the patch configs
patches_branch = 'patches/' + destination
config = get_patch_config(patches_branch)
# Store it
self.store_original_config(config, patches_branch)
# Modify the base so import/export patch works
config['base'] = get_commit_hash(get_current_branch())
# Set it
set_patch_config(patches_branch, config)
def post_patch(self, destination, color='bluef'):
if destination in self.debian_branches:
return
# Tag after patches have been applied
with inbranch(destination):
# Tag
tag_name = self.tag_names[destination]
if tag_exists(tag_name):
if self.interactive:
warning("Tag exists: " + tag_name)
warning("Do you wish to overwrite it?")
if not maybe_continue('y'):
error("Answered no to continue, aborting.", exit=True)
else:
warning("Overwriting tag: " + tag_name)
else:
info("Creating tag: " + tag_name)
execute_command('git tag -f ' + tag_name)
# Report of success
name = destination.split('/')[-1]
package = self.packages[name]
distro = destination.split('/')[-2]
info(ansi(color) + "####" + ansi('reset'), use_prefix=False)
info(ansi(color) + "#### " + ansi('greenf') + "Successfully" +
ansi(color) + " generated '" + ansi('boldon') + distro +
ansi('boldoff') + "' debian for package"
" '" + ansi('boldon') + package.name + ansi('boldoff') + "'" +
" at version '" + ansi('boldon') + package.version + "-" +
str(self.debian_inc) + ansi('boldoff') + "'" + ansi('reset'),
use_prefix=False)
info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False)
def store_original_config(self, config, patches_branch):
with inbranch(patches_branch):
with open('debian.store', 'w+') as f:
f.write(json.dumps(config))
execute_command('git add debian.store')
if has_changes():
execute_command('git commit -m "Store original patch config"')
def generate_ros_distro_diff(track, repository, distro, distro_file_url, distro_file, distro_file_raw):
with inbranch('upstream'):
# Check for package.xml(s)
try:
from catkin_pkg.packages import find_packages
except ImportError:
debug(traceback.format_exc())
error("catkin_pkg was not detected, please install it.",
file=sys.stderr, exit=True)
packages = find_packages(os.getcwd())
if len(packages) == 0:
warning("No packages found, will not generate 'package: path' entries for rosdistro.")
track_dict = get_tracks_dict_raw()['tracks'][track]
last_version = track_dict['last_version']
release_inc = track_dict['release_inc']
if repository not in distro_file['repositories']:
global _user_provided_release_url
distro_file['repositories'][repository] = {'url': _user_provided_release_url or ''}
distro_file['repositories'][repository]['version'] = '{0}-{1}'.format(last_version, release_inc)
if packages and (len(packages) > 1 or packages.keys()[0] != '.'):
distro_file['repositories'][repository]['packages'] = {}
for path, package in packages.iteritems():
if os.path.basename(path) == package.name:
distro_file['repositories'][repository]['packages'][package.name] = None
else:
distro_file['repositories'][repository]['packages'][package.name] = path
distro_file_name = os.path.join('release', distro_file_url.split('/')[-1])
distro_dump = yaml.dump(distro_file, indent=2, default_flow_style=False)
if distro_file_raw != distro_dump:
udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(),
fromfile=distro_file_name, tofile=distro_file_name)
temp_dir = tempfile.mkdtemp()
version = distro_file['repositories'][repository]['version']
udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch')
udiff_raw = ''
info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file))
for line in udiff:
if line.startswith('@@'):
udiff_raw += line
line = fmt('@{cf}' + line)
if line.startswith('+'):
if not line.startswith('+++'):
line += '\n'
udiff_raw += line
line = fmt('@{gf}' + line)
if line.startswith('-'):
if not line.startswith('---'):
line += '\n'
udiff_raw += line
line = fmt('@{rf}' + line)
if line.startswith(' '):
line += '\n'
udiff_raw += line
info(line, use_prefix=False, end='')
with open(udiff_file, 'w+') as f:
f.write(udiff_raw)
return udiff_file, distro_dump
else:
warning("This release resulted in no changes to the ROS distro file...")
return None, None
def store_original_config(self, config, patches_branch):
with inbranch(patches_branch):
with open('rpm.store', 'w+') as f:
f.write(json.dumps(config))
execute_command('git add rpm.store')
if has_changes():
execute_command('git commit -m "Store original patch config"')
def post_rebase(self, destination):
name = destination.split('/')[-1]
# Retrieve the package
package = self.packages[name]
# Handle differently if this is a debian vs distro branch
if destination in self.debian_branches:
info("Placing debian template files into '{0}' branch.".format(
destination))
# Then this is a debian branch
# Place the raw template files
self.place_template_files()
else:
# This is a distro specific debian branch
# Determine the current package being generated
distro = destination.split('/')[-2]
# Create debians for each distro
with inbranch(destination):
data = self.generate_debian(package, distro)
# Create the tag name for later
self.tag_names[destination] = self.generate_tag_name(data)
# Update the patch configs
patches_branch = 'patches/' + destination
config = get_patch_config(patches_branch)
# Store it
self.store_original_config(config, patches_branch)
# Modify the base so import/export patch works
current_branch = get_current_branch()
if current_branch is None:
error("Could not determine current branch.", exit=True)
config['base'] = get_commit_hash(current_branch)
# Set it
set_patch_config(patches_branch, config)
def post_patch(self, destination, color='bluef'):
if destination in self.rpm_branches:
return
# Tag after patches have been applied
with inbranch(destination):
# Tag
tag_name = self.tag_names[destination]
if tag_exists(tag_name):
if self.interactive:
warning("Tag exists: " + tag_name)
warning("Do you wish to overwrite it?")
if not maybe_continue('y'):
error("Answered no to continue, aborting.", exit=True)
else:
warning("Overwriting tag: " + tag_name)
else:
info("Creating tag: " + tag_name)
execute_command('git tag -f ' + tag_name)
# Report of success
name = destination.split('/')[-1]
package = self.packages[name]
distro = destination.split('/')[-2]
info(ansi(color) + "####" + ansi('reset'), use_prefix=False)
info(
ansi(color) + "#### " + ansi('greenf') + "Successfully" +
ansi(color) + " generated '" + ansi('boldon') + distro +
ansi('boldoff') + "' RPM for package"
" '" + ansi('boldon') + package.name + ansi('boldoff') + "'" +
" at version '" + ansi('boldon') + package.version +
"-" + str(self.rpm_inc) + ansi('boldoff') + "'" +
ansi('reset'),
use_prefix=False
)
info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False)
def test_fuerte_package_repository(directory=None):
"""
Release a single catkin stack (fuerte) repository.
"""
directory = directory if directory is not None else os.getcwd()
# Setup
upstream_url = create_upstream_catkin_fuerte_repository('foo', directory)
release_url = create_release_repo(upstream_url, 'git', 'fuerte_devel')
release_dir = os.path.join(directory, 'foo_release_clone')
release_client = VcsClient('git', release_dir)
release_client.checkout(release_url)
with change_directory(release_dir):
###
### Import upstream
###
user('git-bloom-import-upstream --quiet')
# does the upstream branch exist?
assert branch_exists('upstream', local_only=True), "no upstream branch"
# does the upstrea/0.1.0 tag exist?
ret, out, err = user('git tag', return_io=True)
assert out.count('upstream/0.1.0') == 1, "no upstream tag created"
# Is the package.xml from upstream in the upstream branch now?
with inbranch('upstream'):
assert os.path.exists('stack.xml'), \
"upstream did not import: '" + os.getcwd() + "': " + \
str(os.listdir(os.getcwd()))
assert open('stack.xml').read().count('0.1.0'), "not right file"
###
### Release generator
###
with bloom_answer(bloom_answer.ASSERT_NO_QUESTION):
ret = user('git-bloom-generate -y release -s upstream --quiet')
# patch import should have reported OK
assert ret == code.OK, "actually returned ({0})".format(ret)
# do the proper branches exist?
assert branch_exists('release/foo'), "no release/foo branch: " + \
str(get_branches())
assert branch_exists('patches/release/foo'), \
"no patches/release/foo branch"
# was the release tag created?
ret, out, err = user('git tag', return_io=True)
assert out.count('release/foo/0.1.0') == 1, "no release tag created"
###
### Release generator, again
###
with bloom_answer(bloom_answer.ASSERT_NO_QUESTION):
ret = user('git-bloom-generate -y release -s upstream --quiet')
# patch import should have reported OK
assert ret == code.OK, "actually returned ({0})".format(ret)
# do the proper branches exist?
assert branch_exists('release/foo'), "no release/foo branch: " + \
str(get_branches())
assert branch_exists('patches/release/foo'), \
"no patches/release/foo branch"
# was the release tag created?
ret, out, err = user('git tag', return_io=True)
assert out.count('release/foo/0.1.0') == 1, "no release tag created"
def import_patches(patches_path, patches_path_dict, target_branch, version):
info("Overlaying files from patched folder '{0}' on the '{2}' branch into the '{1}' branch..."
.format(patches_path, target_branch, BLOOM_CONFIG_BRANCH))
with inbranch(target_branch):
handle_tree(patches_path_dict, '', patches_path, version)
cmd = ('git commit --allow-empty -m "Overlaid patches from \'{0}\'"'
.format(patches_path))
execute_command(cmd, shell=True)
def write_tracks_dict_raw(tracks_dict, cmt_msg=None, directory=None):
cmt_msg = cmt_msg if cmt_msg is not None else 'Modified tracks.yaml'
with inbranch('bloom'):
with open('tracks.yaml', 'w') as f:
f.write(yaml.dump(tracks_dict, indent=2, default_flow_style=False))
execute_command('git add tracks.yaml', cwd=directory)
execute_command('git commit --allow-empty -m "{0}"'.format(cmt_msg),
cwd=directory)
def non_git_rebase(upstream_branch, directory=None):
# Create a temporary storage directory
tmp_dir = mkdtemp()
# Get the root of the git repository
git_root = get_root(directory)
try:
# Copy the new upstream source into the temporary directory
with inbranch(upstream_branch):
ignores = (".git", ".gitignore", ".svn", ".hgignore", ".hg", "CVS")
parent_source = os.path.join(tmp_dir, "parent_source")
my_copytree(git_root, parent_source, ignores)
# Clear out the local branch
execute_command("git rm -rf *", cwd=directory)
# Collect .* files (excluding .git)
dot_items = []
for item in os.listdir(git_root):
if item in [".git", "..", "."]:
continue
if item.startswith("."):
dot_items.append(item)
# Remove and .* files missed by 'git rm -rf *'
if len(dot_items) > 0:
execute_command("git rm -rf " + " ".join(dot_items), cwd=directory)
# Clear out any untracked files
execute_command("git clean -fdx", cwd=directory) # for good measure?
# Copy the parent source into the newly cleaned directory
my_copytree(parent_source, git_root)
# Commit changes to the repository
execute_command("git add ./*", cwd=directory)
# Collect .* files
dot_items = []
for item in os.listdir(git_root):
if item in [".git", "..", "."]:
continue
if item.startswith("."):
dot_items.append(item)
# Add any .* files missed by 'git add ./*'
if len(dot_items) > 0:
execute_command("git add " + " ".join(dot_items), cwd=directory)
# Remove any straggling untracked files
execute_command("git clean -dXf", cwd=directory)
# Only if we have local changes commit
# (not true if the upstream didn't change any files)
cmd = "git commit "
if not has_changes(directory):
cmd += "--allow-empty "
cmd += "-m \"Rebase from '" + upstream_branch + "'"
if not has_changes(directory):
cmd += " (no changes)"
cmd += '"'
execute_command(cmd, cwd=directory)
finally:
# Clean up
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
def non_git_rebase(upstream_branch, directory=None):
# Create a temporary storage directory
tmp_dir = mkdtemp()
# Get the root of the git repository
git_root = get_root(directory)
try:
# Copy the new upstream source into the temporary directory
with inbranch(upstream_branch):
ignores = ('.git', '.gitignore', '.svn', '.hgignore', '.hg', 'CVS')
parent_source = os.path.join(tmp_dir, 'parent_source')
my_copytree(git_root, parent_source, ignores)
# Clear out the local branch
execute_command('git rm -rf *', cwd=directory)
# Collect .* files (excluding .git)
dot_items = []
for item in os.listdir(git_root):
if item in ['.git', '..', '.']:
continue
if item.startswith('.'):
dot_items.append(item)
# Remove and .* files missed by 'git rm -rf *'
if len(dot_items) > 0:
execute_command('git rm -rf ' + ' '.join(dot_items), cwd=directory)
# Clear out any untracked files
execute_command('git clean -fdx', cwd=directory) # for good measure?
# Copy the parent source into the newly cleaned directory
my_copytree(parent_source, git_root)
# Commit changes to the repository
execute_command('git add ./*', cwd=directory)
# Collect .* files
dot_items = []
for item in os.listdir(git_root):
if item in ['.git', '..', '.']:
continue
if item.startswith('.'):
dot_items.append(item)
# Add any .* files missed by 'git add ./*'
if len(dot_items) > 0:
execute_command('git add ' + ' '.join(dot_items), cwd=directory)
# Remove any straggling untracked files
execute_command('git clean -dXf', cwd=directory)
# Only if we have local changes commit
# (not true if the upstream didn't change any files)
cmd = 'git commit '
if not has_changes(directory):
cmd += '--allow-empty '
cmd += '-m "Rebase from \'' + upstream_branch + "'"
if not has_changes(directory):
cmd += " (no changes)"
cmd += '"'
execute_command(cmd, cwd=directory)
finally:
# Clean up
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
def write_tracks_dict_raw(tracks_dict, cmt_msg=None, directory=None):
upconvert_bloom_to_config_branch()
cmt_msg = cmt_msg if cmt_msg is not None else 'Modified tracks.yaml'
with inbranch(BLOOM_CONFIG_BRANCH):
with open('tracks.yaml', 'w') as f:
f.write(yaml.dump(tracks_dict, indent=2, default_flow_style=False))
execute_command('git add tracks.yaml', cwd=directory)
execute_command('git commit --allow-empty -m "{0}"'.format(cmt_msg),
cwd=directory)
def write_tracks_dict_raw(tracks_dict, cmt_msg=None, directory=None):
upconvert_bloom_to_config_branch()
cmt_msg = cmt_msg if cmt_msg is not None else 'Modified tracks.yaml'
with inbranch(BLOOM_CONFIG_BRANCH):
with open('tracks.yaml', 'w') as f:
f.write(yaml.safe_dump(tracks_dict, indent=2, default_flow_style=False))
execute_command('git add tracks.yaml', cwd=directory)
execute_command('git commit --allow-empty -m "{0}"'.format(cmt_msg),
cwd=directory)
def import_patches(patches_path, patches_path_dict, target_branch, version):
info(
"Overlaying files from patched folder '{0}' on the 'bloom' branch into the '{1}' branch...".format(
patches_path, target_branch
)
)
with inbranch(target_branch):
handle_tree(patches_path_dict, "", patches_path, version)
cmd = "git commit --allow-empty -m \"Overlaid patches from '{0}'\"".format(patches_path)
execute_command(cmd, shell=True)
def set_releaser_history(self, history):
# Assumes that this is called in the target branch
patches_branch = 'patches/' + get_current_branch()
debug("Writing release history to '{0}' branch".format(patches_branch))
with inbranch(patches_branch):
with open('releaser_history.json', 'w') as f:
f.write(json.dumps(history))
execute_command('git add releaser_history.json')
if has_changes():
execute_command('git commit -m "Store releaser history"')
def detect_branches(self):
with inbranch(self.src):
if self.name is not None:
return [self.name]
package_data = get_package_data(self.src)
if type(package_data) not in [list, tuple]:
return package_data
name, version, packages = package_data
self.packages = packages
return name if type(name) is list else [name]
def set_releaser_history(self, history):
# Assumes that this is called in the target branch
patches_branch = 'patches/' + get_current_branch()
debug("Writing release history to '{0}' branch".format(patches_branch))
with inbranch(patches_branch):
with open('releaser_history.json', 'w') as f:
f.write(json.dumps(history))
execute_command('git add releaser_history.json')
if has_changes():
execute_command('git commit -m "Store releaser history"')
def post_patch(self, destination):
# Figure out the version of the given package
with inbranch(destination):
package_data = get_package_data(destination)
if type(package_data) not in [list, tuple]:
return package_data
name, version, packages = package_data
# Execute git tag
execute_command('git tag -f ' + destination + '/' + version)
return 0
def non_git_rebase(upstream_branch, directory=None):
# Create a temporary storage directory
tmp_dir = mkdtemp()
# Get the root of the git repository
git_root = get_root(directory)
try:
# Copy the new upstream source into the temporary directory
with inbranch(upstream_branch):
ignores = ('.git', '.gitignore', '.svn', '.hgignore', '.hg', 'CVS')
parent_source = os.path.join(tmp_dir, 'parent_source')
my_copytree(git_root, parent_source, ignores)
# Clear out any untracked files
execute_command('git clean -fdx', cwd=directory) # for good measure?
# Collect files (excluding .git)
items = []
for item in os.listdir(git_root):
if item in ['.git', '..', '.']:
continue
items.append(item)
# Remove all files
if len(items) > 0:
execute_command('git rm -rf ' + ' '.join(items), cwd=directory)
# Copy the parent source into the newly cleaned directory
my_copytree(parent_source, git_root)
# Commit changes to the repository
execute_command('git add ./*', cwd=directory)
# Collect .* files
dot_items = []
for item in os.listdir(git_root):
if item in ['.git', '..', '.']:
continue
if item.startswith('.'):
dot_items.append(item)
# Add any .* files missed by 'git add ./*'
if len(dot_items) > 0:
execute_command('git add ' + ' '.join(dot_items), cwd=directory)
# Remove any straggling untracked files
execute_command('git clean -dXf', cwd=directory)
# Only if we have local changes commit
# (not true if the upstream didn't change any files)
cmd = 'git commit '
if not has_changes(directory):
cmd += '--allow-empty '
cmd += '-m "Rebase from \'' + upstream_branch + "'"
if not has_changes(directory):
cmd += " (no changes)"
cmd += '"'
execute_command(cmd, cwd=directory)
finally:
# Clean up
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
def detect_branches(self):
self.packages = None
with inbranch(self.src):
if self.name is not None:
self.packages = [self.name]
return [self.name]
name, version, packages = get_package_data(self.src)
self.packages = packages
# Check meta packages for valid CMakeLists.txt
if isinstance(self.packages, dict):
for path, pkg in self.packages.iteritems():
# Check for valid CMakeLists.txt if a metapackage
self.metapackage_check(path, pkg)
return name if type(name) is list else [name]
def post_patch(self, destination):
# Figure out the version of the given package
if self.name is not None:
warning("Cannot automatically tag the release because this is "
"not a catkin project. Please create a tag manually with:")
warning(" git tag -f release/" + str(self.name) + "/<version>")
return 0
with inbranch(destination):
package_data = get_package_data(destination)
if type(package_data) not in [list, tuple]:
return package_data
name, version, packages = package_data
# Execute git tag
execute_command('git tag -f ' + destination + '/' + version)
return 0
def post_patch(self, destination):
# Figure out the version of the given package
if self.name is not None:
warning("""\
Cannot automatically tag the release because this is not a catkin project.""")
warning("""\
Please checkout the release branch and then create a tag manually with:""")
warning(" git checkout release/" + str(self.name))
warning(" git tag -f release/" + str(self.name) + "/<version>")
return
with inbranch(destination):
name, version, packages = get_package_data(destination)
# Execute git tag
release_tag = destination + '/' + version + '-' + self.release_inc
execute_command('git tag ' + release_tag)
def detect_branches(self):
self.packages = None
with inbranch(self.src):
if self.name is not None:
self.packages = [self.name]
return [self.name]
name, version, packages = get_package_data(self.src)
self.packages = packages
# Check meta packages for valid CMakeLists.txt
if isinstance(self.packages, dict):
for path, pkg in self.packages.iteritems():
with change_directory(path):
if is_metapackage(pkg):
check_metapackage_for_valid_cmake(pkg.name)
return name if type(name) is list else [name]
def get_stackage_from_branch(branch):
with inbranch(branch):
package_data = get_package_data(branch)
if type(package_data) not in [list, tuple]:
# It is a ret code
DebianGenerator.exit(package_data)
name, version, packages = package_data
if type(name) is list and len(name) > 1:
error("Debian generator does not support generating "
"from branches with multiple packages in them, use "
"the release generator first to split packages into "
"individual branches.")
DebianGenerator.exit(code.DEBIAN_MULTIPLE_PACKAGES_FOUND)
if type(packages) is dict:
return packages.values()[0], 'package'
return packages, 'stack'
def post_rebase(self, destination):
name = destination.split('/')[-1]
# Retrieve the stackage
stackage, kind = self.packages[name]
# Handle differently if this is a debian vs distro branch
if destination in self.debian_branches:
info("Placing debian template files into '{0}' branch.".format(destination))
# Check for valid CMakeLists.txt if a metapackage
if kind == 'package' and is_metapackage(stackage):
check_metapackage_for_valid_cmake(name)
# Then this is a debian branch
# Place the raw template files
self.place_tempalte_files()
else:
# Check for valid CMakeLists.txt if a metapackage
if kind == 'package' and is_metapackage(stackage):
check_metapackage_for_valid_cmake(name)
# This is a distro specific debian branch
# Determine the current package being generated
distro = destination.split('/')[-2]
### Start debian generation
# Get time of day
from dateutil import tz
stamp = datetime.datetime.now(tz.tzlocal())
# Convert stackage to debian data
data = self.convert_stackage_to_debian_data(stackage, kind)
# Get apt_installer from rosdep
from rosdep2.catkin_support import get_installer
self.apt_installer = get_installer(APT_INSTALLER)
# Create debians for each distro
with inbranch(destination):
self.generate_debian(data, stamp, distro)
# Create the tag name for later
self.tag_names[destination] = self.generate_tag_name(data)
# Update the patch configs
patches_branch = 'patches/' + destination
config = get_patch_config(patches_branch)
# Store it
self.store_original_config(config, patches_branch)
# Modify the base so import/export patch works
current_branch = get_current_branch()
if current_branch is None:
error("Could not determine current branch.", exit=True)
config['base'] = get_commit_hash(current_branch)
# Set it
set_patch_config(patches_branch, config)
def get_package_from_branch(branch):
with inbranch(branch):
try:
package_data = get_package_data(branch)
except SystemExit:
return None
if type(package_data) not in [list, tuple]:
# It is a ret code
RpmGenerator.exit(package_data)
names, version, packages = package_data
if type(names) is list and len(names) > 1:
RpmGenerator.exit("RPM generator does not support generating "
"from branches with multiple packages in them, use "
"the release generator first to split packages into "
"individual branches.")
if type(packages) is dict:
return list(packages.values())[0]
def generate_ros_distro_diff(track, repository, distro, distro_file_url=ROS_DISTRO_FILE):
distro_file_url = distro_file_url.format(distro)
distro_file_raw = fetch_distro_file(distro_file_url)
distro_file = yaml.load(distro_file_raw)
with inbranch('upstream'):
# Check for package.xml(s)
try:
from catkin_pkg.packages import find_packages
except ImportError:
error("catkin_pkg was not detected, please install it.",
file=sys.stderr, exit=True)
packages = find_packages(os.getcwd())
if len(packages) == 0:
warning("No packages found, will not generate 'package: path' entries for rosdistro.")
track_dict = get_tracks_dict_raw()['tracks'][track]
last_version = track_dict['last_version']
release_inc = track_dict['release_inc']
distro_file['repositories'][repository]['version'] = '{0}-{1}'.format(last_version, release_inc)
if packages and (len(packages) > 1 or packages.keys()[0] != '.'):
distro_file['repositories'][repository]['packages'] = {}
for path, package in packages.iteritems():
distro_file['repositories'][repository]['packages'][package.name] = path
distro_file_name = distro_file_url.split('/')[-1]
# distro_dump_orig = yaml.dump(distro_file_orig, indent=2, default_flow_style=False)
distro_dump = yaml.dump(distro_file, indent=2, default_flow_style=False)
udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(),
fromfile=distro_file_name, tofile=distro_file_name)
if udiff:
info("Unified diff for the ROS distro file located at '{0}':".format(distro_file_url))
for line in udiff:
if line.startswith('@@'):
line = fmt('@{cf}' + line)
if line.startswith('+'):
if not line.startswith('+++'):
line += '\n'
line = fmt('@{gf}' + line)
if line.startswith('-'):
if not line.startswith('---'):
line += '\n'
line = fmt('@{rf}' + line)
if line.startswith(' '):
line += '\n'
info(line, use_prefix=False, end='')
else:
warning("This release resulted in no changes to the ROS distro file...")
def get_package_from_branch(branch):
with inbranch(branch):
try:
package_data = get_package_data(branch)
except SystemExit:
return None
if type(package_data) not in [list, tuple]:
# It is a ret code
RpmGenerator.exit(package_data)
names, version, packages = package_data
if type(names) is list and len(names) > 1:
RpmGenerator.exit(
"RPM generator does not support generating "
"from branches with multiple packages in them, use "
"the release generator first to split packages into "
"individual branches.")
if type(packages) is dict:
return list(packages.values())[0]
def detect_branches(self):
self.packages = None
with inbranch(self.src):
if self.name is not None:
self.packages = [self.name]
return [self.name]
package_data = get_package_data(
self.src,
fuerte=(self.rosdistro == 'fuerte'))
if type(package_data) not in [list, tuple]:
return package_data
name, version, packages = package_data
self.packages = packages
# Check meta packages for valid CMakeLists.txt
if isinstance(self.packages, dict):
for path, pkg in self.packages.iteritems():
# Check for valid CMakeLists.txt if a metapackage
self.metapackage_check(path, pkg)
return name if type(name) is list else [name]
def test_create_a_bloom_repository(directory=None):
"""
Create a Bloom Repository:
User creates a new release repository, and calls git-bloom-config new.
actions:
- user creates a folder
- user calls 'git init .' in that folder
- user calls 'git-bloom-config new <track name>'
expects:
- bloom to ask the user if initialization is desired
- bloom prompts the user for input on the settings
- bloom branch to be created
- tracks.yaml to be in bloom branch with appropriate contents
"""
# Setup
user('mkdir new_repo')
user('cd new_repo')
user('git init .')
# Test bloom command
with bloom_answer(['', 'foo', 'https://github.com/bar/foo.git']):
r = user('git-bloom-config new foo', return_io=True, silent=False)
_, out, err = r
assert out.count('ontinue') > 0, \
"git-bloom-config didn't ask about git init:\n```\n" + out + "\n```"
assert branch_exists(BLOOM_CONFIG_BRANCH), \
"branch '{0}' does not exist".format(BLOOM_CONFIG_BRANCH)
with inbranch(BLOOM_CONFIG_BRANCH):
assert os.path.exists('tracks.yaml'), \
"no tracks.yaml file in the 'bloom' branch"
with open('tracks.yaml', 'r') as f:
tracks_dict = yaml.safe_load(f.read())
assert 'tracks' in tracks_dict, "bad bloom configurations"
assert 'foo' in tracks_dict['tracks'], "bad bloom configurations"
track = tracks_dict['tracks']['foo']
assert 'vcs_uri' in track, "bad bloom configurations"
assert 'https://github.com/bar/foo.git' == track['vcs_uri'], \
"bad bloom configurations" + str(track)
# Assert no question on the second attempt
with bloom_answer(bloom_answer.ASSERT_NO_QUESTION):
user('git-bloom-config')
def match_branches_with_prefix(prefix, get_branches, prune=False):
debug("match_branches_with_prefix(" + str(prefix) + ", " +
str(get_branches()) + ")")
branches = []
# Match branches
existing_branches = get_branches()
for branch in existing_branches:
if branch.startswith('remotes/origin/'):
branch = branch.split('/', 2)[-1]
if branch.startswith(prefix):
branches.append(branch)
branches = list(set(branches))
if prune:
# Prune listed branches by packages in latest upstream
with inbranch('upstream'):
pkg_names, version, pkgs_dict = get_package_data('upstream')
for branch in branches:
if branch.split(prefix)[-1].strip('/') not in pkg_names:
branches.remove(branch)
return branches
def match_branches_with_prefix(prefix, get_branches, prune=False):
debug("match_branches_with_prefix(" + str(prefix) + ", " +
str(get_branches()) + ")")
branches = []
# Match branches
existing_branches = get_branches()
for branch in existing_branches:
if branch.startswith('remotes/origin/'):
branch = branch.split('/', 2)[-1]
if branch.startswith(prefix):
branches.append(branch)
branches = list(set(branches))
if prune:
# Prune listed branches by packages in latest upstream
with inbranch('upstream'):
pkg_names, version, pkgs_dict = get_package_data('upstream')
for branch in branches:
if branch.split(prefix)[-1].strip('/') not in pkg_names:
branches.remove(branch)
return branches
def test_create_a_bloom_repository(directory=None):
"""
Create a Bloom Repository:
User creates a new release repository, and calls git-bloom-config new.
actions:
- user creates a folder
- user calls 'git init .' in that folder
- user calls 'git-bloom-config new <track name>'
expects:
- bloom to ask the user if initialization is desired
- bloom prompts the user for input on the settings
- bloom branch to be created
- tracks.yaml to be in bloom branch with appropriate contents
"""
# Setup
user('mkdir new_repo')
user('cd new_repo')
user('git init .')
# Test bloom command
with bloom_answer(['', 'foo', 'https://github.com/bar/foo.git']):
r = user('git-bloom-config new foo', return_io=True, silent=False)
_, out, err = r
assert out.count('ontinue') > 0, \
"git-bloom-config didn't ask about git init:\n```\n" + out + "\n```"
assert branch_exists('bloom'), "branch 'bloom' does not exist"
with inbranch('bloom'):
assert os.path.exists('tracks.yaml'), \
"no tracks.yaml file in the 'bloom' branch"
with open('tracks.yaml', 'r') as f:
tracks_dict = yaml.load(f.read())
assert 'tracks' in tracks_dict, "bad bloom configurations"
assert 'foo' in tracks_dict['tracks'], "bad bloom configurations"
track = tracks_dict['tracks']['foo']
assert 'vcs_uri' in track, "bad bloom configurations"
assert 'https://github.com/bar/foo.git' == track['vcs_uri'], \
"bad bloom configurations" + str(track)
# Assert no question on the second attempt
with bloom_answer(bloom_answer.ASSERT_NO_QUESTION):
user('git-bloom-config')
def get_packages():
with inbranch('upstream'):
_, _, packages = get_package_data('upstream')
return packages
from bloom.git import has_changes
from bloom.util import execute_command
from bloom.util import maybe_continue
print("Generating github pages documentation for version '{0}'...".format(ver))
execute_command('make clean', cwd='doc')
execute_command('python setup.py build_sphinx')
execute_command('sphinxtogithub doc/build/html --verbose')
orig_cwd = os.getcwd()
clone = GitClone()
with clone as clone_dir:
execute_command('git clean -fdx')
with inbranch('gh-pages'):
doc_dir = os.path.join('doc', ver)
if os.path.exists(doc_dir):
warning("Documentation for version '" + ver + "' already exists.")
if not maybe_continue('y'):
sys.exit(-1)
execute_command('git rm -rf ' + doc_dir)
shutil.copytree(os.path.join(orig_cwd, 'doc', 'build', 'html'),
doc_dir)
p = re.compile('\d*[.]\d*[.]\d*')
with open('doc/index.html', 'r') as f:
redirect = f.read()
redirect = p.sub(ver, redirect)
with open('doc/index.html', 'w+') as f:
f.write(redirect)
execute_command('git add -f ' + os.path.join('doc', ver, '*'))
def test_multi_package_repository(directory=None):
"""
Release a multi package catkin (groovy) repository.
"""
directory = directory if directory is not None else os.getcwd()
# Setup
pkgs = ['foo', 'bar_ros', 'baz']
upstream_dir = create_upstream_repository(pkgs,
directory,
format_versions=[1, 2, 3])
upstream_url = 'file://' + upstream_dir
release_url = create_release_repo(upstream_url, 'git', 'groovy_devel',
'groovy')
release_dir = os.path.join(directory, 'foo_release_clone')
release_client = get_vcs_client('git', release_dir)
assert release_client.checkout(release_url)
with change_directory(release_dir):
# First run everything
with bloom_answer(bloom_answer.ASSERT_NO_QUESTION):
cmd = 'git-bloom-release{0} groovy'
if 'BLOOM_VERBOSE' not in os.environ:
cmd = cmd.format(' --quiet')
else:
cmd = cmd.format('')
user(cmd, silent=False)
###
### Import upstream
###
# does the upstream branch exist?
assert branch_exists('upstream', local_only=True), "no upstream branch"
# does the upstrea/0.1.0 tag exist?
ret, out, err = user('git tag', return_io=True)
assert out.count('upstream/0.1.0') == 1, "no upstream tag created"
# Is the package.xml from upstream in the upstream branch now?
with inbranch('upstream'):
for pkg in pkgs:
with change_directory(pkg):
assert os.path.exists(
os.path.join('debian', 'something.udev')), \
"Lost the debian overlaid files in upstream branch"
assert os.path.exists('white space.txt~'), \
"Lost file with whitespace in name in upstream branch"
assert os.path.exists('package.xml'), \
"upstream did not import: " + os.listdir()
with open('package.xml') as f:
assert f.read().count('0.1.0'), "not right file"
###
### Release generator
###
# Check the environment after the release generator
ret, out, err = user('git tag', return_io=True)
for pkg in pkgs:
# Does the release/pkg branch exist?
assert branch_exists('release/groovy/' + pkg), \
"no release/groovy/" + pkg + " branch"
# Does the patches/release/pkg branch exist?
assert branch_exists('patches/release/groovy/' + pkg), \
"no patches/release/groovy/" + pkg + " branch"
# Did the release tag get created?
assert out.count('release/groovy/' + pkg + '/0.1.0-0') == 1, \
"no release tag created for " + pkg
# Is there a package.xml in the top level?
with inbranch('release/groovy/' + pkg):
assert os.path.exists('package.xml'), "release branch invalid"
# Is it the correct package.xml for this pkg?
package_xml = open('package.xml', 'r').read()
assert package_xml.count('<name>' + pkg + '</name>'), \
"incorrect package.xml for " + str(pkg)
# Make a patch
with inbranch('release/groovy/' + pkgs[0]):
user('echo "This is a change" >> README.md')
user('git add README.md')
user('git commit -m "added a readme" --allow-empty')
###
### Release generator, again
###
with bloom_answer(bloom_answer.ASSERT_NO_QUESTION):
ret = user('git-bloom-generate -y rosrelease groovy -s upstream')
# patch import should have reported OK
assert ret == code.OK, "actually returned ({0})".format(ret)
# Check the environment after the release generator
ret, out, err = user('git tag', return_io=True)
for pkg in pkgs:
# Does the release/pkg branch exist?
assert branch_exists('release/groovy/' + pkg), \
"no release/groovy/" + pkg + " branch"
# Does the patches/release/pkg branch exist?
assert branch_exists('patches/release/groovy/' + pkg), \
"no patches/release/groovy/" + pkg + " branch"
# Did the release tag get created?
assert out.count('release/groovy/' + pkg + '/0.1.0-0') == 1, \
"no release tag created for " + pkg
# Is there a package.xml in the top level?
with inbranch('release/groovy/' + pkg):
assert os.path.exists(os.path.join('debian', 'something.udev')), \
"Lost the debian overlaid files in release branch"
assert os.path.exists('white space.txt~'), \
"Lost file with whitespace in name in release branch"
assert os.path.exists('package.xml'), "release branch invalid"
# Is it the correct package.xml for this pkg?
with open('package.xml', 'r') as f:
assert f.read().count('<name>' + pkg + '</name>'), \
"incorrect package.xml for " + str(pkg)
###
### ROSDebian Generator
###
# Check the environment after the release generator
ret, out, err = user('git tag', return_io=True)
for pkg in pkgs:
for distro in ['oneiric', 'precise', 'quantal']:
pkg_san = sanitize_package_name(pkg)
# Does the debian/distro/pkg branch exist?
assert branch_exists('debian/groovy/' + distro + '/' + pkg), \
"no debian/groovy/" + pkg + " branch"
# Does the patches/debian/distro/pkg branch exist?
patches_branch = 'patches/debian/groovy/' + distro + '/' + pkg
assert branch_exists(patches_branch), \
"no " + patches_branch + " branch"
# Did the debian tag get created?
tag = 'debian/ros-groovy-' + pkg_san + '_0.1.0-0_' + distro
assert out.count(tag) == 1, \
"no '" + tag + "'' tag created for '" + pkg + "': `\n" + \
out + "\n`"
# Is there a package.xml in the top level?
with inbranch('debian/groovy/' + distro + '/' + pkg):
assert os.path.exists(
os.path.join('debian', 'something.udev')), \
"Lost the debian overlaid files in debian branch"
assert os.path.exists('white space.txt~'), \
"Lost file with whitespace in name in debian branch"
assert os.path.exists('package.xml'), "debian branch invalid"
# Is there blank lins due to no Conflicts/Replaces?
# See: https://github.com/ros-infrastructure/bloom/pull/329
with open(os.path.join('debian', 'control'), 'r') as f:
assert f.read().count('\n\nHomepage:') == 0, \
"Extra blank line before Homepage detected."
# Is it the correct package.xml for this pkg?
with open('package.xml', 'r') as f:
package_xml = f.read()
assert package_xml.count('<name>' + pkg + '</name>'), \
"incorrect package.xml for " + str(pkg)
format_version = int(
re.search('format="(\d+)"', package_xml).group(1))
# Is there a copyright file for this pkg?
if format_version <= 2:
assert not os.path.exists(
os.path.join('debian', 'copyright')), \
"debian/copyright should not be generated"
else:
with open('debian/copyright', 'r') as f:
assert pkg + ' license' in f.read(), \
"debian/copyright does not include right license text"
def _test_unary_package_repository(release_dir, version, directory=None):
print("Testing in {0} at version {1}".format(release_dir, version))
with change_directory(release_dir):
# First run everything
with bloom_answer(bloom_answer.ASSERT_NO_QUESTION):
cmd = 'git-bloom-release{0} groovy'
if 'BLOOM_VERBOSE' not in os.environ:
cmd = cmd.format(' --quiet')
else:
cmd = cmd.format('')
user(cmd, silent=False)
###
### Import upstream
###
# does the upstream branch exist?
assert branch_exists('upstream', local_only=True), "no upstream branch"
# does the upstrea/<version> tag exist?
ret, out, err = user('git tag', return_io=True)
assert out.count('upstream/' + version) == 1, "no upstream tag created"
# Is the package.xml from upstream in the upstream branch now?
with inbranch('upstream'):
assert os.path.exists('package.xml'), \
"upstream did not import: '" + os.getcwd() + "': " + \
str(os.listdir(os.getcwd()))
assert os.path.exists(os.path.join('debian', 'something.udev')), \
"Lost the debian overlaid files in upstream branch"
assert os.path.exists('white space.txt~'), \
"Lost file with whitespace in name in upstream branch"
with open('package.xml') as f:
package_xml = f.read()
assert package_xml.count(version), "not right file"
###
### Release generator
###
# patch import should have reported OK
assert ret == code.OK, "actually returned ({0})".format(ret)
# do the proper branches exist?
assert branch_exists('release/groovy/foo'), \
"no release/groovy/foo branch"
assert branch_exists('patches/release/groovy/foo'), \
"no patches/release/groovy/foo branch"
# was the release tag created?
ret, out, err = user('git tag', return_io=True)
expected = 'release/groovy/foo/' + version + '-0'
assert out.count(expected) == 1, \
"no release tag created, expected: '{0}'".format(expected)
###
### Make patch
###
with inbranch('release/groovy/foo'):
assert os.path.exists(os.path.join('debian', 'something.udev')), \
"Lost the debian overlaid files in release branch"
assert os.path.exists('white space.txt~'), \
"Lost file with whitespace in name in release branch"
assert os.path.islink(
'include/sym/foo.h'), "Symbolic link lost during pipeline"
if os.path.exists('include/foo.h'):
user('git rm include/foo.h')
else:
if not os.path.exists('include'):
os.makedirs('include')
user('touch include/foo.h')
user('git add include/foo.h')
user('git commit -m "A release patch" --allow-empty')
###
### Test import and export
###
with inbranch('release/groovy/foo'):
export_cmd.export_patches()
remove_cmd.remove_patches()
import_cmd.import_patches()
###
### Release generator, again
###
# patch import should have reported OK
assert ret == code.OK, "actually returned ({0})".format(ret)
# do the proper branches exist?
assert branch_exists('release/groovy/foo'), \
"no release/groovy/foo branch"
assert branch_exists('patches/release/groovy/foo'), \
"no patches/release/groovy/foo branch"
# was the release tag created?
ret, out, err = user('git tag', return_io=True)
assert out.count('release/groovy/foo/' + version) == 1, \
"no release tag created"
def import_tarball(tarball_path, target_branch, version, name):
if tarball_path.endswith('.zip'):
error("Zip archives are not yet supported.", exit=True)
# Create the tarfile handle
targz = tarfile.open(tarball_path, 'r:gz')
with inbranch(target_branch):
# Prepare list of members to extract, ignoring some
ignores = ('.git', '.gitignore', '.svn', '.hgignore', '.hg', 'CVS')
members = targz.getmembers()
members = [m for m in members if m.name.split('/')[-1] not in ignores]
# Clear out the local branch
items = []
for item in os.listdir(os.getcwd()):
if item in ['.git', '..', '.']:
continue
items.append(item)
if len(items) > 0:
execute_command('git rm -rf ' + ' '.join(items))
# Clear out any untracked files
execute_command('git clean -fdx')
# Extract the tarball into the clean branch
targz.extractall(os.getcwd(), members)
# Check for folder nesting (mostly hg)
items = []
for item in os.listdir(os.getcwd()):
if not item.startswith('.'):
items.append(item)
tarball_prefix = os.path.basename(tarball_path)[:-len('.tag.gz')]
if tarball_prefix in items:
debug('Removing nested tarball folder: ' + str(tarball_prefix))
tarball_prefix_path = os.path.join(os.getcwd(), tarball_prefix)
for item in os.listdir(tarball_prefix_path):
if item in ['.', '..']:
continue
item_path = os.path.join(os.getcwd(), tarball_prefix, item)
debug(
'moving ' + str(item_path) + ' to ' +
str(os.path.join(os.getcwd(), item))
)
shutil.move(item_path, os.path.join(os.getcwd(), item))
shutil.rmtree(tarball_prefix_path)
else:
debug('No nested tarball folder found.')
# Commit changes to the repository
items = []
for item in os.listdir(os.getcwd()):
if item in ['.git', '..', '.']:
continue
items.append(item)
if len(items) > 0:
execute_command('git add ' + ' '.join(items))
# Remove any straggling untracked files
execute_command('git clean -dXf')
# Only if we have local changes commit
# (not true if the upstream didn't change any files)
if has_changes():
msg = "Imported upstream version '{0}' of '{1}'"
msg = msg.format(version, name or 'upstream')
cmd = 'git commit -m "{0}"'.format(msg)
execute_command(cmd)
def import_upstream(tarball_path, patches_path, version, name, replace):
# Check for a url and download it
url = urlparse(tarball_path)
if url.scheme: # Some scheme like http, https, or file...
tmp_dir = tempfile.mkdtemp()
try:
info("Fetching file from url: '{0}'".format(tarball_path))
req = load_url_to_file_handle(tarball_path)
tarball_path = os.path.join(tmp_dir, os.path.basename(url.path))
with open(tarball_path, 'wb') as f:
chunk_size = 16 * 1024
while True:
chunk = req.read(chunk_size)
if not chunk:
break
f.write(chunk)
return import_upstream(tarball_path, patches_path, version, name, replace)
finally:
shutil.rmtree(tmp_dir)
# If there is not tarball at the given path, fail
if not os.path.exists(tarball_path):
error("Specified archive does not exists: '{0}'".format(tarball_path),
exit=True)
# If either version or name are not provided, guess from archive name
if not version or not name:
# Parse tarball name
tarball_file = os.path.basename(tarball_path)
ending = None
if tarball_file.endswith('.tar.gz'):
ending = '.tar.gz'
elif tarball_file.endswith('.zip'):
ending = '.zip'
else:
error("Cannot detect type of archive: '{0}'"
.format(tarball_file), exit=True)
tarball_file = tarball_file[:-len(ending)]
split_tarball_file = tarball_file.split('-')
if len(split_tarball_file) < 2 and not version or len(split_tarball_file) < 1:
error("Cannot detect name and/or version from archive: '{0}'"
.format(tarball_file), exit=True)
if not name and len(split_tarball_file) == 1:
name = split_tarball_file[0]
elif not name and len(split_tarball_file) == 1:
name = '-'.join(split_tarball_file[:-1])
if not version and len(split_tarball_file) < 2:
error("Cannot detect version from archive: '{0}'"
.format(tarball_file) + " and the version was not spcified.",
exit=True)
version = version if version else split_tarball_file[-1]
# Check if the patches_path (if given) exists
patches_path_dict = None
if patches_path:
patches_path_dict = ls_tree(BLOOM_CONFIG_BRANCH, patches_path)
if not patches_path_dict:
error("Given patches path '{0}' does not exist in bloom branch."
.format(patches_path), exit=True)
# Do version checking
version_check(version)
# Check for existing tags
upstream_tag = 'upstream/{0}'.format(version)
if tag_exists(upstream_tag):
if not replace:
error("Tag '{0}' already exists, use --replace to override it."
.format(upstream_tag), exit=True)
warning("Removing tag: '{0}'".format(upstream_tag))
delete_tag(upstream_tag)
if not get_git_clone_state():
delete_remote_tag(upstream_tag)
name_tag = '{0}/{1}'.format(name or 'upstream', version)
if name_tag != upstream_tag and tag_exists(name_tag):
if not replace:
error("Tag '{0}' already exists, use --replace to override it."
.format(name_tag), exit=True)
warning("Removing tag: '{0}'".format(name_tag))
delete_tag(name_tag)
if not get_git_clone_state():
delete_remote_tag(name_tag)
# If there is not upstream branch, create one
if not branch_exists('upstream'):
info("Creating upstream branch.")
create_branch('upstream', orphaned=True)
else:
track_branches(['upstream'])
# Import the given tarball
info("Importing archive into upstream branch...")
import_tarball(tarball_path, 'upstream', version, name)
# Handle patches_path
if patches_path:
import_patches(patches_path, patches_path_dict, 'upstream', version)
# Create tags
with inbranch('upstream'):
info("Creating tag: '{0}'".format(upstream_tag))
create_tag(upstream_tag)
if name_tag != upstream_tag:
info("Creating tag: '{0}'".format(name_tag))
create_tag(name_tag)
