def create_repo(pkg_name):
"""Create new gitlab repo"""
ws = Workspace()
ws.recreate_index(write=True)
pkg_list = ws.get_catkin_package_names()
pkg_dicts = ws.get_wstool_packages()
if not pkg_name:
pkg_name = os.path.basename(ws.org_dir)
if pkg_name not in pkg_list:
click.secho(
"{0} does not seem to be a catkin package.".format(pkg_name),
fg="red")
sys.exit(1)
click.echo("Creating repo for {0}".format(pkg_name))
for ps in ws.wstool_config.get_config_elements():
if ps.get_local_name() == pkg_name:
click.secho("Repository has a url already: {0}".format(
ps.get_path_spec().get_uri()))
sys.exit(1)
ws.cd_src()
os.chdir(pkg_name)
git = Git()
ssh_url = git.create_repo(pkg_name)
subprocess.call("git init", shell=True)
subprocess.call("git remote add origin " + ssh_url + " >/dev/null 2>&1",
shell=True)
ws.recreate_index()
click.echo(
"You should run 'mrt maintenance update_url_in_package_xml' now")
def build(pkg_name, this, no_deps, verbose, workspace_docs):
start = time.time()
ws = Workspace()
# add all workspace packages to possibly be doc builded
pkg_list = ws.get_catkin_packages()
is_build_workspace_doc = True
if pkg_name or this:
# build only the specified packge
if pkg_name:
if pkg_name not in pkg_list:
click.secho("Package not found, can't build documentation", fg="red")
sys.exit(1)
# build only this packge
elif this:
pkg_name = os.path.basename(ws.org_dir)
if pkg_name not in pkg_list:
click.secho("{0} does not seem to be a catkin package.".format(pkg_name), fg="red")
sys.exit(1)
# clear doc build package list and set to specified one
pkg_list = {pkg_name: pkg_list[pkg_name]}
# add also dependencies if necessary
if not no_deps:
pkg_list.update(get_dep_packages_in_workspace_(ws, pkg_name))
is_build_workspace_doc = False
# order topologically to get the build order right
pkg_list = catkin_pkg.topological_order.topological_order_packages(pkg_list)
# output build summary
end = time.time()
click.secho("[doc build] Found '{}' packages in {:.1f} seconds.".format(len(pkg_list), end - start))
# build packages
for pkg, _ in pkg_list:
output_start_status_(pkg)
build_(ws, pkg, verbose)
output_finished_states_(pkg)
# build workspace docs
if workspace_docs:
click.echo("[doc build] Build workspace documentation")
output_start_status_("workspace_doc")
build_workspace_doc_(ws)
output_finished_states_("workspace_doc")
def clean(pkg_name):
if pkg_name:
pkg_list = [pkg_name]
else:
ws = Workspace()
pkg_list = ws.get_catkin_package_names()
for pkg in pkg_list:
_, _, package_doxygen_output_dir = check_paths_(pkg)
if os.path.exists(package_doxygen_output_dir):
shutil.rmtree(package_doxygen_output_dir)
if not pkg_name:
doc_folder = get_workspace_doc_folder_(ws)
if os.path.exists(doc_folder):
shutil.rmtree(doc_folder)
def restore(name):
"""Restore a catkin workspace from a snapshot"""
org_dir = os.getcwd()
filename = os.path.join(org_dir, name)
workspace = os.path.join(org_dir, os.path.basename(name).split(".")[0] + "_snapshot_ws")
# Read archive
try:
zf = zipfile.ZipFile(filename, "r", zipfile.ZIP_DEFLATED)
# file_list = [f.filename for f in zf.filelist]
version = zf.read(user_settings['Snapshot']['VERSION_FILE'])
except IOError:
click.echo(os.getcwd())
click.secho("Can't find file: '" + name + user_settings['Snapshot']['FILE_ENDING'] + "'", fg="red")
sys.exit()
if version == "0.1.0":
# Create workspace folder
try:
os.mkdir(workspace)
os.chdir(workspace)
ws = Workspace(quiet=True)
ws.create()
except OSError:
click.secho("Directory {0} exists already".format(workspace), fg="red")
os.chdir(org_dir)
sys.exit(1)
# Extract archive
zf.extractall(path=workspace)
os.remove(os.path.join(workspace, user_settings['Snapshot']['VERSION_FILE']))
# Clone packages
click.secho("Cloning packages", fg="green")
ws.load()
ws.update()
ws.resolve_dependencies()
# Build workspace
click.secho("Building workspace", fg="green")
subprocess.call(["catkin", "clean", "-a"])
subprocess.call(["catkin", "build"])
else:
click.secho("ERROR: Snapshot version not known.", fg="red")
def update_rosinstall():
"""Reinitialise the workspace index"""
ws = Workspace()
ws.cd_src()
click.secho("Removing wstool database src/.rosinstall", fg="yellow")
os.remove(".rosinstall")
click.echo("Initializing wstool...")
ws.recreate_index(write=True)
def show(pkg_name):
if pkg_name:
index_file = get_html_index_file_path_(pkg_name)
else:
ws = Workspace()
doc_folder = get_workspace_doc_folder_(ws)
index_file = os.path.join(doc_folder, "html", "index.html")
if not os.path.isfile(index_file):
build_(ws, pkg_name)
if not os.path.isfile(index_file):
raise RuntimeError("Documentation output not found. Expected to be in " + index_file)
webbrowser.open("file://" + index_file)
def create(name):
"""Create a snapshot of the current workspace."""
suffix = "_" + time.strftime("%y%m%d")
snapshot_name = name + suffix + user_settings['Snapshot']['FILE_ENDING']
filename = os.path.join(os.getcwd(), snapshot_name)
# First test whether it's safe to create a snapshot
ws = Workspace()
ws.test_for_changes(prompt="Are you sure you want to continue? These changes won't be included in the snapshot!")
# Create snapshot of rosinstall
ws.cd_root()
ws.snapshot(filename=".rosinstall")
# Create archive
with open(user_settings['Snapshot']['VERSION_FILE'], "w") as f:
f.write(user_settings['Snapshot']['SNAPSHOT_VERSION'])
files = [('.rosinstall', 'src/.rosinstall'), '.catkin_tools', user_settings['Snapshot']['VERSION_FILE']]
files += [os.path.join(dp, f) for dp, dn, fn in os.walk(".catkin_tools") for f in fn]
zip_files(files, filename)
os.remove(".rosinstall")
os.remove(user_settings['Snapshot']['VERSION_FILE'])
click.secho("Wrote snapshot to " + filename, fg="green")
def main(action, resolve_deps, eclipse, debug, release, verbose, warnings,
no_warnings, default_yes, catkin_args):
""" A wrapper for catkin. """
org_dir = os.getcwd()
ws = Workspace()
ws.cd_root()
build_eclipse = False
if action == "build":
if debug:
catkin_args = ("-DCMAKE_BUILD_TYPE=Debug", ) + catkin_args
elif release:
catkin_args = ("-DCMAKE_BUILD_TYPE=RelWithDebInfo", ) + catkin_args
else:
catkin_args = ("-DCMAKE_BUILD_TYPE={}".format(
user_settings['Catkin']['DEFAULT_BUILD_TYPE']), ) + catkin_args
if (user_settings['Catkin']['SHOW_WARNINGS_DURING_COMPILATION']
or warnings) and not no_warnings:
catkin_args = ("-DCMAKE_CXX_FLAGS=-Wall", ) + (
"-DCMAKE_CXX_FLAGS=-Wextra", ) + catkin_args
if eclipse:
build_eclipse = True
catkin_args = ("--force-cmake", ) + catkin_args
catkin_args = ("-GEclipse CDT4 - Unix Makefiles", ) + catkin_args
if verbose:
catkin_args += ("-v", )
catkin_args += ("--make-args VERBOSE=1 --", )
if resolve_deps:
ws.resolve_dependencies(default_yes=default_yes)
os.chdir(org_dir)
process = subprocess.Popen(["catkin", action] + list(catkin_args))
process.wait() # Wait for process to finish and set returncode
if build_eclipse:
set_eclipse_project_setting(ws.root)
sys.exit(process.returncode)
def update_cmakelists(package, this):
"""Update CMAKELISTS"""
ws = Workspace()
catkin_packages = ws.get_catkin_package_names()
# Read in newest CMakeLists.txt
current_version = None
# download newest version:
click.echo("Downloading newest template from gitlab")
git = Git()
mrt_build_repo = git.find_repo("mrt_build")
new_cmakelists = git.server.getrawfile(
mrt_build_repo['id'], "master", 'mrt_tools/templates/CMakeLists.txt')
for line in new_cmakelists.splitlines():
if line.startswith("#pkg_version="):
current_version = line[:-1]
break
if not current_version:
click.secho("current pkg_version could not be found.", fg='red')
sys.exit(1)
if this:
package = os.path.basename(ws.org_dir)
if package not in catkin_packages:
click.secho(
"{0} does not seem to be a catkin package.".format(package),
fg="red")
sys.exit(1)
if not package:
for pkg_name in catkin_packages:
ws.cd_src()
check_and_update_cmakelists(pkg_name, current_version)
else:
ws.cd_src()
check_and_update_cmakelists(package, current_version)
def main(ctx):
"""Package related tasks...
:param ctx:
"""
ctx.obj = Workspace()
def add_user(git):
"""Add a user to a repository"""
click.echo("Loading... please wait a moment")
users = list(git.server.getall(git.server.getusers))
users = sorted(users, key=lambda k: k['name'])
repo_dicts = git.get_repos()
repo_dicts = sorted(repo_dicts, key=lambda k: k['path_with_namespace'])
user_choice, _ = get_user_choice([user["name"] for user in users],
prompt="Please choose a user")
user = users[user_choice]
click.echo("--> Selected user '{}'\n".format(user['name']))
repo_choice, _ = get_user_choice(
[repo["path_with_namespace"] for repo in repo_dicts],
prompt="Please choose a repo.")
repo = repo_dicts[repo_choice]
click.echo("--> Selected repo '{}'\n".format(repo["name_with_namespace"]))
roles = ["Guest", "Reporter", "Developer", "Master", "Owner"]
_, role = get_user_choice(roles,
prompt='Please choose a role for the user.',
default=2)
click.confirm("\nAdd user {0} to repo {1} with role {2}?\n".format(
user["name"].upper(), repo["path_with_namespace"].upper(),
role.upper()),
default=True,
abort=True)
git.server.addprojectmember(repo["id"], user["id"], role)
if not click.confirm("Should I test dependencies?", default=True):
return
# Create temporary workspace
org_dir = os.getcwd()
if os.path.exists("/tmp/mrtgitlab_test_ws"):
shutil.rmtree("/tmp/mrtgitlab_test_ws")
os.mkdir("/tmp/mrtgitlab_test_ws")
os.chdir("/tmp/mrtgitlab_test_ws")
ws = Workspace(quiet=True)
ws.create()
# Clone pkg and resolve dependencies
pkg_name = repo["name"]
gl_repo = git.find_repo(pkg_name) # Gives error string
ws.add(pkg_name, gl_repo[git.get_url_string()])
ws.resolve_dependencies(git=git)
# Read in dependencies
ws.load()
new_repos = ws.get_catkin_packages()
new_repos.pop(pkg_name)
click.echo("\n\nFound following new repos:")
for r in new_repos:
click.echo("- {}".format(r))
for r in new_repos:
if click.confirm("\nAdd user {0} to repo {1} aswell?".format(
user["name"].upper(), r.upper()),
default=True):
# Add user as well
click.echo("\nAdding user {0} to repo {1}\n".format(
user["name"].upper(), r.upper()))
repo_id = [s["id"] for s in repo_dicts if s["name"] == r]
_, role = get_user_choice(
roles,
prompt='Please choose a role for the user for this repo.',
default=2)
git.server.addprojectmember(repo_id[0], user["id"], role)
os.chdir(org_dir)
shutil.rmtree("/tmp/mrtgitlab_test_ws")
def main(action, args):
"""
A wrapper for wstool.
"""
ws = Workspace()
ws.cd_src()
if action == "init":
click.secho("Removing wstool database src/.rosinstall", fg="yellow")
os.remove(".rosinstall")
click.echo("Initializing wstool...")
ws.recreate_index()
return
# Need to init wstool?
if not os.path.exists(".rosinstall"):
ws.write()
if action == "update":
# Test git credentials to avoid several prompts
if ws.contains_https():
test_git_credentials()
ws.recreate_index() # Rebuild .rosinstall in case a package was deletetd manually
# Speedup the pull process by parallelization
if not [a for a in args if a.startswith("-j")]:
args += ("-j10",)
if action == "fetch":
if ws.contains_https():
test_git_credentials()
action = "foreach"
args = ("git fetch", )
if action == "push":
# Search for unpushed commits
unpushed_repos = ws.unpushed_repos()
if len(unpushed_repos) > 0:
if click.confirm("Push them now?"):
for x in unpushed_repos:
ws.cd_src()
os.chdir(x)
subprocess.call("git push", shell=True)
else:
click.echo("No unpushed commits.")
sys.exit(0)
# Pass the rest to wstool
if len(args) == 0:
process = subprocess.Popen(["wstool", action, "-t", ws.src])
else:
process = subprocess.Popen(["wstool", action, "-t", ws.src] + list(args))
process.wait() # Wait for process to finish and set returncode
if action == "status":
ws.unpushed_repos()
sys.exit(process.returncode)
def update_url_in_package_xml():
"""Updates missing or wrong URL into package.xml"""
def insert_url(file_name, url):
with open(file_name, 'r') as f:
contents = f.readlines()
click.clear()
for index, item in enumerate(contents):
click.echo("{0}: {1}".format(index, item[:-1]))
linenumber = click.prompt(
"\n\nPlease specify the line to insert the url in", type=click.INT)
contents.insert(linenumber,
' <url type="repository">{0}</url>\n'.format(url))
contents = "".join(contents)
with open(file_name, 'w') as f:
f.write(contents)
click.clear()
if click.confirm("OK, did that. Commit these changes?"):
org_dir = os.getcwd()
os.chdir(os.path.dirname(file_name))
subprocess.call("git add {0}".format(file_name), shell=True)
subprocess.call(
"git commit -m 'Added repository url to package.xml'",
shell=True)
os.chdir(org_dir)
ws = Workspace()
ws.catkin_pkg_names = ws.get_catkin_package_names()
ws.config = wstool_config.Config([], ws.src)
ws.cd_src()
for pkg_name in ws.catkin_pkg_names:
filename = os.path.join(ws.src, pkg_name, "package.xml")
# Try reading it from git repo
try:
# TODO Maybe try to always get the https/ssh url? Right now, it is only checked against how YOU have it
# configured.
with open(pkg_name + "/.git/config", 'r') as f:
git_url = next(line[7:-1] for line in f
if line.startswith("\turl"))
except (IOError, StopIteration):
git_url = None
# Try to read it from package xml
try:
if len(ws.catkin_pkgs[pkg_name].urls) > 1:
raise IndexError
xml_url = ws.catkin_pkgs[pkg_name].urls[0].url
except IndexError:
xml_url = None
# Testing all cases:
if xml_url is not None and git_url is not None:
if xml_url != git_url:
click.secho(
"WARNING in {0}: URL declared in src/{1}/package.xml, differs from the git repo url for {"
"0}!".format(pkg_name.upper(), pkg_name),
fg="red")
click.echo("PackageXML: {0}".format(xml_url))
click.echo("Git repo : {0}".format(git_url))
if click.confirm(
"Replace the url in package.xml with the correct one?"
):
subprocess.call(
"sed -i -e '/ <url/d' {0}".format(filename),
shell=True)
insert_url(filename, git_url)
if xml_url is not None and git_url is None:
click.secho(
"WARNING in {0}: URL declared in package.xml, but {1} does not seem to be a remote "
"repository!".format(pkg_name.upper(), pkg_name),
fg="yellow")
if click.confirm("Remove the url in package.xml?"):
click.secho("Fixing...", fg="green")
subprocess.call("sed -i -e '/ <url/d' {0}".format(filename),
shell=True)
if xml_url is None and git_url is not None:
click.secho(
"WARNING in {0}: No URL (or multiple) defined in package.xml!".
format(pkg_name.upper()),
fg="yellow")
if click.confirm(
"Insert (Replace) the url in package.xml with the correct one?"
):
subprocess.call("sed -i -e '/ <url/d' {0}".format(filename),
shell=True)
insert_url(filename, git_url)
if xml_url is None and git_url is None:
click.secho(
"INFO in {0}: Does not seem to be a git repository. You should use Version Control for your "
"code!".format(pkg_name.upper()),
fg="cyan")
if git_url is not None:
ws.add(pkg_name, git_url, update=False)
ws.write()
def rename_pkg(new_name):
""" """
ws = Workspace()
package = os.path.basename(ws.org_dir)
catkin_packages = ws.get_catkin_package_names()
if package not in catkin_packages:
click.secho(
"{0} does not seem to be a catkin package.".format(package),
fg="red")
sys.exit(1)
if new_name in catkin_packages:
click.secho(
"{0} does already exist in your workspace.".format(new_name),
fg="red")
sys.exit(1)
# Test files
for dirName, subdirList, fileList in os.walk(ws.src + "/" + package):
if "/.git" in dirName:
continue
for fname in fileList:
if fname.endswith(".h") or fname.endswith(".hh") or fname.endswith(".hpp") \
or fname.endswith(".cc") or fname.endswith(".cpp"):
# Adjust includes
subprocess.call("sed -i -e 's:#include\s[\"<]" + package +
"/:#include \"" + new_name + "/:g' " +
dirName + "/" + fname,
shell=True)
else:
# Rename all other occurrences
subprocess.call("sed -i -e 's/" + package + "/" + new_name +
"/g' " + dirName + "/" + fname,
shell=True)
# Move include folder
if os.path.exists(ws.src + "/" + package + "/include/" + package):
shutil.move(ws.src + "/" + package + "/include/" + package,
ws.src + "/" + package + "/include/" + new_name)
# Test for git repo
if not os.path.exists(ws.src + "/" + package + "/.git"):
click.echo("Renamed package " + package + " to " + new_name)
return
os.chdir(ws.src + "/" + package)
click.echo("The following files in this package have been changed:")
subprocess.call("git status -s", shell=True)
click.echo("")
click.echo("Next steps:")
click.echo("\t-Review changes")
click.echo("\t-Commit changes")
click.echo("")
while ws.test_for_changes(package, quiet=True):
click.prompt("Continue, when changes are commited and pushed...")
click.echo("")
click.confirm("Do you want to move the gitlab project now?", abort=True)
click.echo("Moving gitlab project...")
git = Git()
project = git.find_repo(package)
namespace = project["namespace"]["name"]
project_id = project["id"]
if not git.server.editproject(project_id, name=new_name, path=new_name):
click.secho("There was a problem, moving the project. Aborting!",
fg="red")
sys.exit(1)
click.echo("Updating git remote...")
os.chdir(ws.src + "/" + package)
project = git.find_repo(new_name, namespace)
new_url = project[git.get_url_string()]
subprocess.call("git remote set-url origin " + new_url +
" >/dev/null 2>&1",
shell=True)
click.echo("Updating local ws...")
ws.cd_src()
shutil.move(package, new_name)
os.chdir(new_name)
os.remove(ws.src + "/.rosinstall")
ws.recreate_index(write=True)
click.echo("")
click.echo("Next steps:")
click.echo("\t-Adjust includes in other packages")
def ws(local):
ws = Workspace()
ws_root = ws.get_root()
ws_src = os.path.join(ws_root, "src")
if not os.path.exists(ws_src):
raise Exception("Cannot find workspace source folder")
if local:
mrt_tools.DockerCheck.ws(ws_root)
return
ssh = _connect()
# copy data to server
sftp = ssh.open_sftp()
# create docker folder if not exist
sftp.chdir("/tmp")
if "docker" not in sftp.listdir():
sftp.mkdir("docker")
sftp.chmod("docker",
mode=stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IRUSR
| stat.S_IWUSR | stat.S_IXUSR)
sftp.chdir("docker")
# create temporary file and create tar archive
try:
tarPathName = tempfile.mkstemp()[1]
tarFile = tarfile.TarFile(tarPathName, "w")
tempFileName = os.path.basename(tarPathName)
remote_work_dir = os.path.join("/tmp", "docker", tempFileName)
# prepare wokspace
sftp.mkdir(tempFileName,
mode=stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
_executeSshCommand(
ssh, 'bash -c "cd {0} && mrt ws init"'.format(remote_work_dir))
os.chdir(ws_src)
tarFile.add(".",
exclude=lambda p: os.path.basename(os.path.abspath(p)).
startswith('.'))
tarFile.close()
# copy tar archive to remote
dest_tar = os.path.join(remote_work_dir, tempFileName + ".tar")
sftp.put(tarPathName, dest_tar)
finally:
if os.path.exists(tarPathName):
os.remove(tarPathName)
sftp.close()
remote_work_src = os.path.join(remote_work_dir, "src")
ssh.exec_command("tar -xf " + dest_tar + " -C " + remote_work_src)
ssh.exec_command("rm " + dest_tar)
# run docker build
execLine = 'bash -c "cd {0} && mrt check ws --local"'.format(
remote_work_dir)
_executeSshCommand(ssh, execLine)
def main(ctx):
"""A collection of tools to perform on a catkin workspace"""
if ctx.invoked_subcommand == "init":
ctx.obj = Workspace(quiet=True)
else:
ctx.obj = Workspace()
from mrt_tools.Workspace import Workspace
from mrt_tools.utilities import *
import catkin_pkg.topological_order
import os
import subprocess
import tempfile
import webbrowser
import hashlib
import time
# Autocompletion
try:
tmp_ws = Workspace(quiet=True)
suggestions = tmp_ws.get_catkin_package_names()
repo_list = import_repo_names()
os.chdir(tmp_ws.org_dir)
except:
suggestions = []
repo_list = []
self_dir = get_script_root()
########################################################################################################################
# Package
########################################################################################################################
@click.group(short_help="Build and show the documentation of a package.",
help="The documentation of a package is build with doxygen using a template Doxyfile. If you wish, "
"you can also provide an individual Doxyfile by placing it in the root folder of the package. "