def add(ws, pkg_names):
"""Clone catkin packages from gitlab."""
"""
This tool searches for, and clones a package from the Gitlab Server into the current workspace.
Execute this script from within a catkin workspace
"""
git = Git()
for pkg_name in pkg_names:
# Test for package
if ws.find(pkg_name):
click.echo(
"Package {0} already present in workspace config. Run 'mrt wstool update'."
.format(pkg_name))
continue
# Add package to workspace
repo = git.find_repo(pkg_name)
if not repo:
continue
url = repo[git.get_url_string()]
ws.add(pkg_name, url)
ws.resolve_dependencies(git=git)
def create_repo(pkg_name):
"""Create new gitlab repo"""
ws = Workspace()
ws.recreate_index(write=True)
pkg_list = ws.get_catkin_package_names()
pkg_dicts = ws.get_wstool_packages()
if not pkg_name:
pkg_name = os.path.basename(ws.org_dir)
if pkg_name not in pkg_list:
click.secho(
"{0} does not seem to be a catkin package.".format(pkg_name),
fg="red")
sys.exit(1)
click.echo("Creating repo for {0}".format(pkg_name))
for ps in ws.wstool_config.get_config_elements():
if ps.get_local_name() == pkg_name:
click.secho("Repository has a url already: {0}".format(
ps.get_path_spec().get_uri()))
sys.exit(1)
ws.cd_src()
os.chdir(pkg_name)
git = Git()
ssh_url = git.create_repo(pkg_name)
subprocess.call("git init", shell=True)
subprocess.call("git remote add origin " + ssh_url + " >/dev/null 2>&1",
shell=True)
ws.recreate_index()
click.echo(
"You should run 'mrt maintenance update_url_in_package_xml' now")
def create(ws, pkg_name, pkg_type, ros, create_git_repo):
""" Create a new catkin package """
ws.cd_root()
pkg_name = check_naming(pkg_name)
if ros:
pkg_name += "_ros"
if pkg_type == "exec":
pkg_name += "_tool"
if pkg_name in get_rosdeps():
click.secho(
"This name collides with a rosdep dependency. Please choose a different one.",
fg="red")
sys.exit(1)
click.echo("Creating package with name.... " + pkg_name)
click.echo(" --> Package type.... " + pkg_type)
if ros:
click.echo(" --> Create ROS Package.... YES")
else:
click.echo(" --> Create ROS Package.... NO")
if create_git_repo:
click.echo(" --> Create gitlab repository.... YES")
else:
click.echo(" --> Create gitlab repository.... NO")
user = get_gituserinfo()
click.echo(" --> Package Maintainer.... " + user['name'] + " <" +
user['email'] + ">")
create_directories(pkg_name, pkg_type, ros)
create_files(pkg_name, pkg_type, ros)
if create_git_repo:
git = Git()
url = git.create_repo(pkg_name)
subprocess.call("sed -i " + "-e 's#\${PACKAGE_REPOSITORY_URL}#" + url +
"#g' " + "package.xml",
shell=True)
# Initialize repository
ws.cd_src()
os.chdir(pkg_name)
subprocess.call("git init", shell=True)
subprocess.call("git remote add origin " + url + " >/dev/null 2>&1",
shell=True)
with open('.gitignore', 'w') as f:
f.write("*~")
subprocess.call("git add . >/dev/null 2>&1", shell=True)
subprocess.call("git commit -m 'Initial commit' >/dev/null 2>&1",
shell=True)
subprocess.call("git push -u origin master >/dev/null 2>&1",
shell=True)
ws.add(pkg_name, url)
else:
subprocess.call("sed -i -e '/ <url/d' package.xml", shell=True)
def update_repo_cache(quiet):
"""
Read repo list from server and write it into caching file.
:rtype : object
"""
# Because we are calling this during autocompletion, we don't wont any errors.
# -> Just exit when something is not ok.
error_occurred = False
try:
# Connect
git = Git(quiet=quiet)
repo_dicts = git.get_repos()
if not repo_dicts:
raise Exception
if not quiet:
click.echo("Update was successful")
except:
# In case the connection didn't succeed, the file is going to be flushed -> we don't seem to have a
# connection anyway and don't want old data.
if not quiet:
click.echo("There was an error during update.")
error_occurred = True
repo_dicts = []
# Remove lock file, so that it will soon be tried again.
try:
os.remove(user_settings['Cache']['CACHE_LOCK_FILE'])
except OSError:
pass
dir_name = os.path.dirname(user_settings['Cache']['CACHE_FILE'])
if not os.path.exists(dir_name):
os.makedirs(dir_name)
with open(user_settings['Cache']['CACHE_FILE'], "w") as f:
if error_occurred:
f.write("AN_ERROR_OCCURRED, CACHING_WAS_UNSUCCESSFUL,")
else:
for r in repo_dicts:
f.write(r["name"] + ",")
def update_cached_deps():
git = Git()
if os.path.exists(user_settings['Cache']['CACHED_DEPS_WS']):
click.echo("Removing existing files in workspace")
shutil.rmtree(user_settings['Cache']['CACHED_DEPS_WS'])
os.makedirs(user_settings['Cache']['CACHED_DEPS_WS'])
click.echo("Retrieving repo list")
repo_list = list(git.server.getall(git.server.getprojects, per_page=100))
click.echo("Downloading package.xml files")
skipped_repos = []
with click.progressbar(repo_list) as repos:
branches = []
for repo in repos:
errcount = 0
branches = git.server.getbranches(repo['id'])
while not branches and errcount < 5:
branches = git.server.getbranches(repo['id'])
errcount += 1
if errcount == 5:
skipped_repos.append(repo['name'])
continue
for branch in branches:
# TODO Test checksum before downloading?
file_contents = git.server.getrawfile(repo['id'],
branch['name'],
'package.xml')
if file_contents is None or file_contents is False:
skipped_repos.append(repo['name'])
continue
file_name = os.path.join(
user_settings['Cache']['CACHED_DEPS_WS'],
repo['namespace']['name'], repo['name'], branch['name'],
'package.xml')
if not os.path.exists(os.path.dirname(file_name)):
os.makedirs(os.path.dirname(file_name))
with open(file_name, 'w') as f:
f.write(file_contents)
skipped_repos = set(skipped_repos)
click.echo("Skipped the following repos:")
for repo in skipped_repos:
click.echo("- {}".format(repo))
def reset():
click.confirm("Deleting userdata first. Continue?", abort=True)
delete_credentials()
username = getpass.getuser()
name = click.prompt("Please enter your first and last name")
email = click.prompt("Please enter your email address")
username = click.prompt("Please enter your Gitlab username",
default=username)
password = click.prompt("Please enter your Gitlab password",
hide_input=True)
set_gituserinfo(name=name, email=email)
credentialManager.store('username', username)
credentialManager.store('password', password)
Git()
def update_cmakelists(package, this):
"""Update CMAKELISTS"""
ws = Workspace()
catkin_packages = ws.get_catkin_package_names()
# Read in newest CMakeLists.txt
current_version = None
# download newest version:
click.echo("Downloading newest template from gitlab")
git = Git()
mrt_build_repo = git.find_repo("mrt_build")
new_cmakelists = git.server.getrawfile(
mrt_build_repo['id'], "master", 'mrt_tools/templates/CMakeLists.txt')
for line in new_cmakelists.splitlines():
if line.startswith("#pkg_version="):
current_version = line[:-1]
break
if not current_version:
click.secho("current pkg_version could not be found.", fg='red')
sys.exit(1)
if this:
package = os.path.basename(ws.org_dir)
if package not in catkin_packages:
click.secho(
"{0} does not seem to be a catkin package.".format(package),
fg="red")
sys.exit(1)
if not package:
for pkg_name in catkin_packages:
ws.cd_src()
check_and_update_cmakelists(pkg_name, current_version)
else:
ws.cd_src()
check_and_update_cmakelists(package, current_version)
def delete_credentials():
# Remove saved credentials
credentialManager.delete('username')
credentialManager.delete('password')
credentialManager.delete('token')
# Remove cache
if os.path.exists(os.path.expanduser("~/.git-credential-cache/socket")):
os.remove(os.path.expanduser("~/.git-credential-cache/socket"))
# Remove git info
subprocess.call("git config --global --remove-section user", shell=True)
# Remove SSH Key
sshkeys = Git.get_local_ssh_keys()
if sshkeys:
click.secho(
"You have an ssh key stored on this machine. If you want to remove ALL of your userdata, "
"please delete it manually.",
fg="yellow")
for nr, key in enumerate(sshkeys):
click.echo("\t" + str(nr + 1) + ")\t" + key.path)
def permissions(ctx):
"""Tools for handling permissions..."""
ctx.obj = Git()
def create_token():
"""Create new gitlab token"""
credentialManager.delete('token')
Git()
def show(ctx):
"""Display a list of..."""
ctx.obj = Git()
def rename_pkg(new_name):
""" """
ws = Workspace()
package = os.path.basename(ws.org_dir)
catkin_packages = ws.get_catkin_package_names()
if package not in catkin_packages:
click.secho(
"{0} does not seem to be a catkin package.".format(package),
fg="red")
sys.exit(1)
if new_name in catkin_packages:
click.secho(
"{0} does already exist in your workspace.".format(new_name),
fg="red")
sys.exit(1)
# Test files
for dirName, subdirList, fileList in os.walk(ws.src + "/" + package):
if "/.git" in dirName:
continue
for fname in fileList:
if fname.endswith(".h") or fname.endswith(".hh") or fname.endswith(".hpp") \
or fname.endswith(".cc") or fname.endswith(".cpp"):
# Adjust includes
subprocess.call("sed -i -e 's:#include\s[\"<]" + package +
"/:#include \"" + new_name + "/:g' " +
dirName + "/" + fname,
shell=True)
else:
# Rename all other occurrences
subprocess.call("sed -i -e 's/" + package + "/" + new_name +
"/g' " + dirName + "/" + fname,
shell=True)
# Move include folder
if os.path.exists(ws.src + "/" + package + "/include/" + package):
shutil.move(ws.src + "/" + package + "/include/" + package,
ws.src + "/" + package + "/include/" + new_name)
# Test for git repo
if not os.path.exists(ws.src + "/" + package + "/.git"):
click.echo("Renamed package " + package + " to " + new_name)
return
os.chdir(ws.src + "/" + package)
click.echo("The following files in this package have been changed:")
subprocess.call("git status -s", shell=True)
click.echo("")
click.echo("Next steps:")
click.echo("\t-Review changes")
click.echo("\t-Commit changes")
click.echo("")
while ws.test_for_changes(package, quiet=True):
click.prompt("Continue, when changes are commited and pushed...")
click.echo("")
click.confirm("Do you want to move the gitlab project now?", abort=True)
click.echo("Moving gitlab project...")
git = Git()
project = git.find_repo(package)
namespace = project["namespace"]["name"]
project_id = project["id"]
if not git.server.editproject(project_id, name=new_name, path=new_name):
click.secho("There was a problem, moving the project. Aborting!",
fg="red")
sys.exit(1)
click.echo("Updating git remote...")
os.chdir(ws.src + "/" + package)
project = git.find_repo(new_name, namespace)
new_url = project[git.get_url_string()]
subprocess.call("git remote set-url origin " + new_url +
" >/dev/null 2>&1",
shell=True)
click.echo("Updating local ws...")
ws.cd_src()
shutil.move(package, new_name)
os.chdir(new_name)
os.remove(ws.src + "/.rosinstall")
ws.recreate_index(write=True)
click.echo("")
click.echo("Next steps:")
click.echo("\t-Adjust includes in other packages")
def test_setup_git():
cm.credentialManager = cm.BaseCredentialManager()
cm.credentialManager.store("username", "root")
cm.credentialManager.store("password", "gitlab_root")
user_settings['Gitlab']['HOST_URL'] = "http://localhost:10080"
git = Git(quiet=True)
assert cm.credentialManager.get_token() is not None
# Cleanup
repos = git.get_repos()
for repo in repos:
git.server.deleteproject(repo["id"])
keys = git.server.getsshkeys()
for key in keys:
git.server.deletesshkey(key["id"])
# Test read operations
assert git.get_namespaces() == {"root": 0}
assert git.find_repo("testrepo") is None
# Create a repo
time.sleep(1)
click.prompt = lambda text, default=None, hide_input=False, confirmation_prompt=False, type=None, \
prompt_suffix=': ', value_proc=None, show_default=True, err=False: 0
assert git.create_repo(
"testrepo") == "http://localhost:10080/root/testrepo.git"
assert git.find_repo("testrepo") is not None
# Test ssh
assert git.check_ssh_key() is False
git.ssh_key = SSHkey(name="mrt_test_key")
if os.path.exists(git.ssh_key.path):
os.remove(git.ssh_key.path)
git.ssh_key.create()
git.upload_ssh_key()
assert git.check_ssh_key() is True
# Test git credential cache
test_git_credentials()
def resolve_dependencies(self, git=None, default_yes=None):
# TODO maybe use rosdep2 package directly
click.echo("Resolving dependencies...")
# Test whether ros is sourced
if is_ros_sourced() is False:
click.secho(
"ROS_ROOT not set. Run: 'source /opt/ros/$ROS_DISTRO/setup.bash'",
fg="red")
sys.exit(1)
if changed_base_yaml():
click.secho("Base YAML file changed, running 'rosdep update'.",
fg="green")
subprocess.call("rosdep update", shell=True)
if not git:
git = Git()
regex_rosdep_resolve = re.compile(
"ERROR\[([^\]]*)\]: Cannot locate rosdep definition for \[([^\]]*)\]"
)
while True:
rosdep_process = subprocess.Popen(
['rosdep', 'check', '--from-paths', self.src, '--ignore-src'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
rosdep_output, rosdep_err = rosdep_process.communicate()
if not rosdep_err:
break
missing_packages = dict()
for match in regex_rosdep_resolve.finditer(rosdep_err):
missing_packages[match.group(2)] = match.group(1)
if not missing_packages:
click.echo(rosdep_output)
click.echo(rosdep_err)
sys.exit(1)
gitlab_packages = []
for missing_package, package_dep_specified in missing_packages.items(
):
# Search for package in gitlab
repo = git.find_repo(missing_package)
if repo:
url = repo[git.get_url_string()]
self.add(missing_package, url, update=True)
gitlab_packages.append(missing_package)
else:
# no Gitlab project found
if not self.updated_apt:
# first not found package. Update apt-get and ros.
click.secho(
"Updating mrt apt-get and rosdep and resolve again. This might take a while ...",
fg='green')
update_apt_and_ros_packages()
self.updated_apt = True
break
else:
click.secho(
"Package {0} (requested from: {1}) could not be found."
.format(missing_package, package_dep_specified),
fg='red')
sys.exit(1)
# Load new gitlab packages
self.write()
# install missing system dependencies
if default_yes:
subprocess.check_call([
"rosdep", "install", "--default-yes", "--from-paths", self.src,
"--ignore-src"
])
else:
subprocess.check_call([
"rosdep", "install", "--from-paths", self.src, "--ignore-src"
])