def get_yaml_from_uri(uri):
"""reads and parses yaml from a local file or remote uri"""
stream = None
try:
try:
if os.path.isfile(uri):
stream = open(uri, 'r')
else:
stream = urlopen(uri)
except IOError as ioe:
raise MultiProjectException(
"Is not a local file, nor able to download as a URL [%s]: %s\n" % (uri, ioe))
except ValueError as vae:
raise MultiProjectException(
"Is not a local file, nor a valid URL [%s] : %s\n" % (uri, vae))
if not stream:
raise MultiProjectException("couldn't load config uri %s\n" % uri)
try:
yamldata = yaml.load(stream)
except yaml.YAMLError as yame:
raise MultiProjectException(
"Invalid multiproject yaml format in [%s]: %s\n" % (uri, yame))
# we want a list or a dict, but pyyaml parses xml as string
if type(yamldata) == 'str':
raise MultiProjectException(
"Invalid multiproject yaml format in [%s]: %s\n" % (uri, yamldata))
finally:
if stream is not None:
stream.close()
return yamldata
def insert_element(self, new_config_elt, merge_strategy='KillAppend'):
"""
Insert ConfigElement to self.trees, checking for duplicate
local-name or path first. In case local_name matches, follow
given strategy
- KillAppend (default): remove old element, append new at the end
- MergeReplace: remove first such old element, insert new at that position.
- MergeKeep: Discard new element
In case local path matches but local name does not, raise Exception
:returns: the action performed None, 'Append', 'KillAppend', 'MergeReplace', 'MergeKeep'"""
removals = []
replaced = False
for index, loop_elt in enumerate(self.trees):
# if paths are os.path.realpath, no symlink problems.
relationship = realpath_relation(loop_elt.get_path(),
new_config_elt.get_path())
if relationship == 'SAME_AS':
if os.path.normpath(loop_elt.get_local_name()) != os.path.normpath(new_config_elt.get_local_name()):
raise MultiProjectException("Elements with different local_name target the same path: %s, %s" % (loop_elt, new_config_elt))
else:
if (loop_elt == new_config_elt):
return None
if (merge_strategy == 'MergeReplace' or
(merge_strategy == 'KillAppend' and
index == len(self.trees) - 1)):
self.trees[index] = new_config_elt
# keep looping to check for overlap when replacing non-scm with scm entry
replaced = True
if (loop_elt.is_vcs_element or not new_config_elt.is_vcs_element):
return 'MergeReplace'
elif merge_strategy == 'KillAppend':
removals.append(loop_elt)
elif merge_strategy == 'MergeKeep':
return 'MergeKeep'
else:
raise LookupError(
"No such merge strategy: %s" % str(merge_strategy))
elif ((relationship == 'CHILD_OF' and new_config_elt.is_vcs_element())
or (relationship == 'PARENT_OF' and loop_elt.is_vcs_element())):
# we do not allow any elements to be children of scm elements
# to allow for parallel updates and because rosinstall may
# delete scm folders on update, and thus subfolders can be
# deleted with their parents
raise MultiProjectException(
"Managed Element paths overlap: %s, %s" % (loop_elt,
new_config_elt))
if replaced:
return 'MergeReplace'
for loop_elt in removals:
self.trees.remove(loop_elt)
self.trees.append(new_config_elt)
if len(removals) > 0:
return 'KillAppend'
return 'Append'
def install(self,
checkout=True,
backup=True,
backup_path=None,
inplace=False,
verbose=False):
"""
Runs the equivalent of SCM checkout for new local repos or
update for existing.
:param checkout: whether to use an update command or
a checkout/clone command
:param backup: if checkout is True and folder exists,
if backup is false folder will be DELETED.
:param backup_path: if checkout is true and backup is true,
move folder to this location
:param inplace: for symlinks, allows to delete contents
at target location and checkout to there.
"""
if checkout is True:
print("[%s] Fetching %s (version %s) to %s" %
(self.get_local_name(), self.uri, self.version,
self.get_path()))
if self.path_exists():
if os.path.islink(self.path):
if inplace is False:
# remove same as unlink
os.remove(self.path)
else:
shutil.rmtree(os.path.realpath(self.path))
else:
if backup is False:
shutil.rmtree(self.path)
else:
self.backup(backup_path)
if not self._get_vcsc().checkout(
self.uri, self.version, verbose=verbose):
raise MultiProjectException(
"[%s] Checkout of %s version %s into %s failed." %
(self.get_local_name(), self.uri, self.version,
self.get_path()))
else:
print("[%s] Updating %s" %
(self.get_local_name(), self.get_path()))
if not self._get_vcsc().update(self.version, verbose=verbose):
raise MultiProjectException(
"[%s] Update Failed of %s" %
(self.get_local_name(), self.get_path()))
print("[%s] Done." % self.get_local_name())
def prepare_install(self,
backup_path=None,
arg_mode='abort',
robust=False):
preparation_report = PreparationReport(self)
present = self.detect_presence()
if present or self.path_exists():
# Directory exists see what we need to do
error_message = None
if not present:
error_message = "Failed to detect %s presence at %s." % (
self.get_vcs_type_name(), self.path)
else:
cur_url = self._get_vcsc().get_url()
if cur_url is not None:
# strip trailing slashes for #3269
cur_url = cur_url.rstrip('/')
if not cur_url or cur_url != self.uri.rstrip('/'):
# local repositories get absolute pathnames
if not (os.path.isdir(self.uri) and os.path.isdir(cur_url)
and samefile(cur_url, self.uri)):
if not self._get_vcsc().url_matches(cur_url, self.uri):
error_message = "Url %s does not match %s requested." % (
cur_url, self.uri)
if error_message is None:
# update should be possible
preparation_report.checkout = False
else:
# If robust ala continue-on-error, just error now and
# it will be continued at a higher level
if robust:
raise MultiProjectException("Update Failed of %s: %s" %
(self.path, error_message))
# prompt the user based on the error code
if arg_mode == 'prompt':
print("Prepare updating %s (version %s) to %s" %
(self.uri, self.version, self.path))
mode = Ui.get_ui().prompt_del_abort_retry(error_message,
allow_skip=True)
else:
mode = arg_mode
if mode == 'backup':
preparation_report.backup = True
if backup_path is None:
print("Prepare updating %s (version %s) to %s" %
(self.uri, self.version, self.path))
preparation_report.backup_path = Ui.get_ui(
).get_backup_path()
else:
preparation_report.backup_path = backup_path
if mode == 'abort':
preparation_report.abort = True
preparation_report.error = error_message
if mode == 'skip':
preparation_report.skip = True
preparation_report.error = error_message
if mode == 'delete':
preparation_report.backup = False
return preparation_report
def cmd_diff(self, target_path, argv, config=None):
parser = OptionParser(usage="usage: rosws diff [localname]* ",
description=__MULTIPRO_CMD_DICT__["diff"],
epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
# required here but used one layer above
parser.add_option("-t", "--target-workspace", dest="workspace",
default=None,
help="which workspace to use",
action="store")
(_, args) = parser.parse_args(argv)
if config is None:
config = multiproject_cmd.get_config(
target_path,
additional_uris=[],
config_filename=self.config_filename)
elif config.get_base_path() != target_path:
raise MultiProjectException(
"Config path does not match %s %s " % (config.get_base_path(),
target_path))
if len(args) > 0:
difflist = multiproject_cmd.cmd_diff(config, localnames=args)
else:
difflist = multiproject_cmd.cmd_diff(config)
alldiff = []
for entrydiff in difflist:
if entrydiff['diff'] is not None and entrydiff['diff'] != '':
alldiff.append(entrydiff['diff'])
print('\n'.join(alldiff))
return False
def aggregate_from_uris(config_uris, config_filename=None, allow_other_element=True):
"""
Builds a List of PathSpec from a list of location strings (uri,
paths). If locations is a folder, attempts to find config_filename
in it, and use "folder/config_filename" instead(rewriting element
path and stripping scm nature), else add folder as PathSpec.
Anything else, parse yaml at location, and add a PathSpec for each
element.
:param config_uris: source of yaml
:param config_filename: file to use when given a folder
:param allow_other_element: if False, discards elements
to be added without SCM information
"""
aggregate_source_yaml = []
# build up a merged list of config elements from all given config_uris
if config_uris is None:
return []
for loop_uri in config_uris:
source_path_specs = get_path_specs_from_uri(
loop_uri, config_filename)
# allow duplicates, dealt with in Config class
if not allow_other_element:
for spec in source_path_specs:
if not spec.get_scmtype():
raise MultiProjectException(
"Forbidden non-SCM element: %s (%s)" %
(spec.get_local_name(), spec.get_legacy_type()))
aggregate_source_yaml.extend(source_path_specs)
return aggregate_source_yaml
def add_uris(config,
additional_uris,
merge_strategy="KillAppend",
allow_other_element=True):
"""
changes the given config by merging with the additional_uris
:param config: a Config objects
:param additional_uris: the location of config specifications or folders
:param config_filename: name of files which may be looked at for config
information
:param merge_strategy: One of 'KillAppend, 'MergeKeep', 'MergeReplace'
:param allow_other_element: if False, discards elements to be added with
no SCM information
:returns: a dict {<local-name>: (<action>, <path-spec>), <local-name>: ...}
determined by the merge_strategy
:raises MultiProjectException: on plenty of errors
"""
if config is None:
raise MultiProjectException("Need to provide a Config.")
if not additional_uris:
return {}
if config.get_config_filename() is None:
added_uris = additional_uris
else:
added_uris = []
# reject if the additional uri points to the same file as our
# config is based on
for uri in additional_uris:
# check whether we try to merge with other workspace
comp_uri = None
if (os.path.isfile(uri)
and os.path.basename(uri) == config.get_config_filename()):
# add from other workspace by file
comp_uri = os.path.dirname(uri)
if (os.path.isdir(uri) and os.path.isfile(
os.path.join(uri, config.get_config_filename()))):
# add from other workspace by dir
comp_uri = uri
if (comp_uri is not None and realpath_relation(
os.path.abspath(comp_uri),
os.path.abspath(config.get_base_path())) == 'SAME_AS'):
print(
'Warning: Discarding config basepath as additional uri: %s'
% uri)
continue
added_uris.append(uri)
actions = {}
if len(added_uris) > 0:
path_specs = aggregate_from_uris(added_uris,
config.get_config_filename(),
allow_other_element)
for path_spec in path_specs:
action = config.add_path_spec(path_spec, merge_strategy)
actions[path_spec.get_local_name()] = (action, path_spec)
return actions
def install(self,
checkout=True,
backup=False,
backup_path=None,
robust=False,
verbose=False):
if not self.install_success:
raise MultiProjectException("Unittest Mock says install failed")
def _get_vcsc(self):
# lazy initializer
if self.vcsc is None:
try:
self.vcsc = get_vcs_client(self._scmtype, self.get_path())
except VcsError as exc:
raise MultiProjectException(
"Unable to create vcs client of type %s for %s: %s" %
(self._scmtype, self.get_path(), exc))
return self.vcsc
def _create_vcs_config_element(self, scmtype, path, local_name, uri, version='', properties=None):
try:
eclass = self.registry[scmtype]
except LookupError:
raise MultiProjectException(
"No VCS client registered for vcs type %s" % scmtype)
return eclass(scmtype=scmtype,
path=path,
local_name=local_name,
uri=uri,
version=version,
properties=properties)
def backup(self, backup_path):
if not backup_path:
raise MultiProjectException(
"[%s] Cannot install %s. backup disabled." %
(self.get_local_name(), self.get_path()))
backup_path = os.path.join(
backup_path,
"%s_%s" % (os.path.basename(self.path),
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")))
print("[%s] Backing up %s to %s" %
(self.get_local_name(), self.get_path(), backup_path))
shutil.move(self.path, backup_path)
def cmd_regenerate(self, target_path, argv, config=None):
parser = OptionParser(
usage="usage: %s regenerate" % self.progname,
formatter=IndentedHelpFormatterWithNL(),
description=__MULTIPRO_CMD_DICT__["remove"] + """
this command without options generates files setup.sh, setup.bash and
setup.zsh. Note that doing this is unnecessary in general, as these
files do not change anymore, unless you change from one ROS distro to
another (which you should never do like this, create a separate new
workspace instead), or you deleted or modified any of those files
accidentally.
""",
epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
parser.add_option("-c",
"--catkin",
dest="catkin",
default=False,
help="Declare this is a catkin build.",
action="store_true")
parser.add_option("--cmake-prefix-path",
dest="catkinpp",
default=None,
help="Where to set the CMAKE_PREFIX_PATH",
action="store")
# -t option required here for help but used one layer above, see cli_common
parser.add_option("-t",
"--target-workspace",
dest="workspace",
default=None,
help="which workspace to use",
action="store")
(options, args) = parser.parse_args(argv)
if len(args) > 0:
print("Error: Too many arguments.")
print(parser.usage)
return -1
if config is None:
config = get_config(target_path,
additional_uris=[],
config_filename=self.config_filename)
elif config.get_base_path() != target_path:
raise MultiProjectException("Config path does not match %s %s " %
(config.get_base_path(), target_path))
rosinstall_cmd.cmd_generate_ros_files(config,
target_path,
nobuild=True,
rosdep_yes=False,
catkin=options.catkin,
catkinpp=options.catkinpp,
no_ros_allowed=True)
return 0
def install(self,
checkout=True,
backup=True,
backup_path=None,
verbose=False):
"""
Runs the equivalent of SCM checkout for new local repos or
update for existing.
:param checkout: whether to use an update command or
a checkout/clone command
:param backup: if checkout is True and folder exists,
if backup is false folder will be DELETED.
:param backup_path: if checkout is true and backup is true,
move folder to this location
"""
if checkout is True:
print("[%s] Installing %s (version %s) to %s" %
(self.get_local_name(), self.uri, self.version,
self.get_path()))
if self.path_exists():
if (backup is False):
shutil.rmtree(self.path)
else:
self.backup(backup_path)
if not self._get_vcsc().checkout(
self.uri, self.version, verbose=verbose):
raise MultiProjectException(
"[%s] Checkout of %s version %s into %s failed." %
(self.get_local_name(), self.uri, self.version,
self.get_path()))
else:
print("[%s] Updating %s" %
(self.get_local_name(), self.get_path()))
if not self._get_vcsc().update(self.version, verbose=verbose):
raise MultiProjectException(
"[%s] Update Failed of %s" %
(self.get_local_name(), self.get_path()))
print("[%s] Done." % self.get_local_name())
def do_work(self):
localname = ""
scm = None
uri = ""
curr_uri = None
exists = False
version = "" # what is given in config file
modified = ""
actualversion = "" # revision number of version
specversion = "" # actual revision number
localname = self.element.get_local_name()
path = self.element.get_path() or localname
if localname is None or localname == "":
raise MultiProjectException(
"Missing local-name in element: %s" % self.element)
abs_path = normabspath(path, self.path)
if (os.path.exists(abs_path)):
exists = True
if self.element.is_vcs_element():
if not exists:
path_spec = self.element.get_path_spec()
version = path_spec.get_version()
else:
path_spec = self.element.get_versioned_path_spec()
version = path_spec.get_version()
curr_uri = path_spec.get_curr_uri()
status = self.element.get_status(self.path)
if (status is not None and status.strip() != ''):
modified = True
specversion = path_spec.get_revision()
if (version is not None and version.strip() != '' and
(specversion is None or specversion.strip() == '')):
specversion = '"%s"' % version
actualversion = path_spec.get_current_revision()
scm = path_spec.get_scmtype()
uri = path_spec.get_uri()
return {
'scm': scm,
'exists': exists,
'localname': localname,
'path': path,
'uri': uri,
'curr_uri': curr_uri,
'version': version,
'specversion': specversion,
'actualversion': actualversion,
'modified': modified,
'properties': self.element.get_properties()
}
def __init__(self,
path_specs,
install_path,
config_filename=None,
extended_types=None,
merge_strategy='KillAppend'):
"""
:param config_source_dict: A list (e.g. from yaml) describing the config, list of dict, each dict describing one element.
:param config_filename: When given a folder, Config
:param merge_strategy: how to deal with entries with equivalent path. See insert_element
will look in folder for file of that name for more config source, str.
"""
assert install_path is not None, "Install path is None"
if path_specs is None:
raise MultiProjectException("Passed empty source to create config")
# All API operations must grant that elements in trees have unique local_name and paths
# Also managed (VCS) entries must be disjunct (meaning one cannot be in a child folder of another managed one)
# The idea is that managed entries can safely be concurrently modified
self.trees = []
self.base_path = os.path.abspath(install_path)
self.config_filename = None
if config_filename is not None:
self.config_filename = os.path.basename(config_filename)
# using a registry primarily for unit test design
self.registry = {
'svn': AVCSConfigElement,
'git': AVCSConfigElement,
'hg': AVCSConfigElement,
'bzr': AVCSConfigElement,
'tar': AVCSConfigElement
}
if extended_types is not None:
self.registry = dict(
list(self.registry.items()) + list(extended_types.items()))
for path_spec in path_specs:
action = self.add_path_spec(path_spec, merge_strategy)
# Usual action in init should be 'Append', anything else is unusual
if action == 'KillAppend':
print("Replace existing entry %s by appending." %
path_spec.get_local_name())
elif action == 'MergeReplace':
print("Replace existing entry %s" % path_spec.get_local_name())
elif action == 'MergeKeep':
print("Keep existing entry %s, discard later one" %
path_spec.get_local_name())
def _insert_vcs_path_spec(self, path_spec, local_path, merge_strategy='KillAppend'):
# Get the version and source_uri elements
source_uri = normalize_uri(path_spec.get_uri(), self.get_base_path())
version = path_spec.get_version()
try:
local_name = os.path.normpath(path_spec.get_local_name())
elem = self._create_vcs_config_element(path_spec.get_scmtype(),
local_path,
local_name,
source_uri,
version,
properties=path_spec.get_tags())
return self.insert_element(elem, merge_strategy)
except LookupError as ex:
raise MultiProjectException("Abstracted VCS Config failed. Exception: %s" % ex)
def __init__(self, path, local_name, uri, version='', properties=None):
"""
Creates a config element for a VCS repository.
:param path: absolute or relative path, str
:param vcs_client: Object compatible with vcstools.VcsClientBase
:param local_name: display name for the element, str
:param uri: VCS uri to checkout/pull from, str
:param version: optional revision spec (tagname, SHAID, ..., str)
"""
super(VCSConfigElement, self).__init__(path, local_name, properties)
if uri is None:
raise MultiProjectException(
"Invalid scm entry having no uri attribute for path %s" % path)
# strip trailing slashes if defined to not be too strict #3061
self.uri = uri.rstrip('/')
self.version = version
def get_config(basepath,
additional_uris=None,
config_filename=None,
merge_strategy='KillAppend'):
"""
Create a Config element necessary for all other commands. The
command will look at the uris in sequence, each can be a web
resource, a filename or a folder. In case it is a folder, when a
config_filename is provided, the folder will be searched for a
file of that name, and that one will be used. Else the folder
will be considered a target location for the config. All files
will be parsed for config elements, thus conceptually the input to
Config is an expanded list of config elements. Config takes this
list and consolidates duplicate paths by keeping the last one in
the list.
:param basepath: where relative paths shall be resolved against
:param additional_uris: the location of config specifications or folders
:param config_filename: name of files which may be looked at for config information
:param merge_strategy: One of 'KillAppend, 'MergeKeep', 'MergeReplace'
:returns: a Config object
:raises MultiProjectException: on plenty of errors
"""
if basepath is None:
raise MultiProjectException("Need to provide a basepath for Config.")
#print("source...........................", path_specs)
## Generate the config class with the uri and path
if (config_filename is not None and basepath is not None
and os.path.isfile(os.path.join(basepath, config_filename))):
base_path_specs = get_path_specs_from_uri(os.path.join(
basepath, config_filename),
as_is=True)
else:
base_path_specs = []
config = Config(base_path_specs,
basepath,
config_filename=config_filename,
merge_strategy=merge_strategy)
add_uris(config, additional_uris, merge_strategy)
return config
def add_uris(config, additional_uris, merge_strategy="KillAppend"):
"""
changes the given config by merging with the additional_uris
:param config: a Config objects
:param additional_uris: the location of config specifications or folders
:param config_filename: name of files which may be looked at for config information
:param merge_strategy: One of 'KillAppend, 'MergeKeep', 'MergeReplace'
:returns: a dict {<local-name>: (<action>, <path-spec>), <local-name>: ...} determined by the merge_strategy
:raises MultiProjectException: on plenty of errors
"""
if config is None:
raise MultiProjectException("Need to provide a Config.")
if additional_uris is None or len(additional_uris) == 0:
return {}
added_uris = []
if config.get_config_filename() is not None:
for uri in additional_uris:
comp_uri = None
if (os.path.isfile(uri)
and os.path.basename(uri) == config.get_config_filename()):
comp_uri = os.path.dirname(uri)
if (os.path.isdir(uri) and os.path.isfile(
os.path.join(uri, config.get_config_filename()))):
comp_uri = uri
if (comp_uri is not None and realpath_relation(
os.path.abspath(comp_uri),
os.path.abspath(config.get_base_path())) == 'SAME_AS'):
print(
'Warning: Discarding config basepath as additional uri: %s'
% uri)
continue
added_uris.append(uri)
path_specs = aggregate_from_uris(added_uris, config.get_config_filename())
actions = {}
for path_spec in path_specs:
action = config.add_path_spec(path_spec, merge_strategy)
actions[path_spec.get_local_name()] = (action, path_spec)
return actions
def cmd_remove(self, target_path, argv, config=None):
parser = OptionParser(
usage="usage: %s remove [localname]*" % self.progname,
formatter=IndentedHelpFormatterWithNL(),
description=__MULTIPRO_CMD_DICT__["remove"] + """
The command removes entries from your configuration file, it does not affect your filesystem.
""",
epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
(_, args) = parser.parse_args(argv)
if len(args) < 1:
print("Error: Too few arguments.")
print(parser.usage)
return -1
if config is None:
config = multiproject_cmd.get_config(
target_path,
additional_uris=[],
config_filename=self.config_filename)
elif config.get_base_path() != target_path:
raise MultiProjectException("Config path does not match %s %s " %
(config.get_base_path(), target_path))
success = True
elements = select_elements(config, args)
for element in elements:
if not config.remove_element(element.get_local_name()):
success = False
print(
"Bug: No such element %s in config, aborting without changes"
% (element.get_local_name()))
break
if success:
print("Overwriting %s" %
os.path.join(config.get_base_path(), self.config_filename))
shutil.move(
os.path.join(config.get_base_path(),
self.config_filename), "%s.bak" %
os.path.join(config.get_base_path(), self.config_filename))
self.config_generator(config, self.config_filename)
print("Removed entries %s" % args)
return 0
def cmd_status(self, target_path, argv, config=None):
parser = OptionParser(
usage="usage: %s status [localname]* " % self.progname,
description=__MULTIPRO_CMD_DICT__["status"] +
". The status columns meanings are as the respective SCM defines them.",
epilog="""See: http://www.ros.org/wiki/rosinstall for details""")
parser.add_option("--untracked",
dest="untracked",
default=False,
help="Also shows untracked files",
action="store_true")
# -t option required here for help but used one layer above, see cli_common
parser.add_option("-t",
"--target-workspace",
dest="workspace",
default=None,
help="which workspace to use",
action="store")
(options, args) = parser.parse_args(argv)
if config is None:
config = multiproject_cmd.get_config(
target_path,
additional_uris=[],
config_filename=self.config_filename)
elif config.get_base_path() != target_path:
raise MultiProjectException("Config path does not match %s %s " %
(config.get_base_path(), target_path))
if len(args) > 0:
statuslist = multiproject_cmd.cmd_status(
config, localnames=args, untracked=options.untracked)
else:
statuslist = multiproject_cmd.cmd_status(
config, untracked=options.untracked)
allstatus = []
for entrystatus in statuslist:
if entrystatus['status'] is not None:
allstatus.append(entrystatus['status'])
print(''.join(allstatus), end='')
return 0
def prompt_merge(target_path,
additional_uris,
additional_specs,
path_change_message=None,
merge_strategy='KillAppend',
confirmed=False,
confirm=False,
show_advanced=True,
show_verbosity=True,
config_filename=None,
config=None,
allow_other_element=True):
"""
Prompts the user for the resolution of a merge. Without
further options, will prompt only if elements change. New
elements are just added without prompt.
:param target_path: Location of the config workspace
:param additional_uris: uris from which to load more elements
:param additional_specs: path specs for additional elements
:param path_change_message: Something to tell the user about elements order
:param merge_strategy: See Config.insert_element
:param confirmed: Never ask
:param confirm: Always ask, supercedes confirmed
:param config: None or a Config object for target path if available
:param show_advanced: if true allow to change merge strategy
:param show_verbosity: if true allows to change verbosity
:param allow_other_element: if False merge fails hwen it could cause other elements
:returns: tupel (Config or None if no change, bool path_changed)
"""
if config is None:
config = multiproject_cmd.get_config(target_path,
additional_uris=[],
config_filename=config_filename)
elif config.get_base_path() != target_path:
msg = "Config path does not match %s %s " % (config.get_base_path(),
target_path)
raise MultiProjectException(msg)
local_names_old = [
x.get_local_name() for x in config.get_config_elements()
]
extra_verbose = confirmed or confirm
abort = False
last_merge_strategy = None
while not abort:
if (last_merge_strategy is None
or last_merge_strategy != merge_strategy):
if not config_filename:
# should never happen right now with rosinstall/rosws/wstool
# TODO Need a better way to work with clones of original config
raise ValueError('Cannot merge when no config filename is set')
newconfig = multiproject_cmd.get_config(
target_path,
additional_uris=[],
config_filename=config_filename)
config_actions = multiproject_cmd.add_uris(
newconfig,
additional_uris=additional_uris,
merge_strategy=merge_strategy,
allow_other_element=allow_other_element)
for path_spec in additional_specs:
action = newconfig.add_path_spec(path_spec, merge_strategy)
config_actions[path_spec.get_local_name()] = (action,
path_spec)
last_merge_strategy = merge_strategy
local_names_new = [
x.get_local_name() for x in newconfig.get_config_elements()
]
path_changed = False
ask_user = False
output = ""
new_elements = []
changed_elements = []
discard_elements = []
for localname, (action, new_path_spec) in list(config_actions.items()):
index = -1
if localname in local_names_old:
index = local_names_old.index(localname)
if action == 'KillAppend':
ask_user = True
if (index > -1 and local_names_old[:index + 1]
== local_names_new[:index + 1]):
action = 'MergeReplace'
else:
changed_elements.append(
_get_element_diff(new_path_spec, config,
extra_verbose))
path_changed = True
if action == 'Append':
path_changed = True
new_elements.append(
_get_element_diff(new_path_spec, config, extra_verbose))
elif action == 'MergeReplace':
changed_elements.append(
_get_element_diff(new_path_spec, config, extra_verbose))
ask_user = True
elif action == 'MergeKeep':
discard_elements.append(
_get_element_diff(new_path_spec, config, extra_verbose))
ask_user = True
if len(changed_elements) > 0:
output += "\n Change details of element (Use --merge-keep or --merge-replace to change):\n"
if extra_verbose:
output += " %s\n" % ("\n".join(sorted(changed_elements)))
else:
output += " %s\n" % (", ".join(sorted(changed_elements)))
if len(new_elements) > 0:
output += "\n Add new elements:\n"
if extra_verbose:
output += " %s\n" % ("\n".join(sorted(new_elements)))
else:
output += " %s\n" % (", ".join(sorted(new_elements)))
if local_names_old != local_names_new[:len(local_names_old)]:
old_order = ' '.join(reversed(local_names_old))
new_order = ' '.join(reversed(local_names_new))
output += "\n %s " % path_change_message or "Element order change"
output += "(Use --merge-keep or --merge-replace to prevent) "
output += "from\n %s\n to\n %s\n\n" % (old_order, new_order)
ask_user = True
if output == "":
return (None, False)
if not confirm and (confirmed or not ask_user):
print(" Performing actions: ")
print(output)
return (newconfig, path_changed)
else:
print(output)
showhelp = True
while (showhelp):
showhelp = False
prompt = "Continue: (y)es, (n)o"
if show_verbosity:
prompt += ", (v)erbosity"
if show_advanced:
prompt += ", (a)dvanced options"
prompt += ": "
mode_input = Ui.get_ui().get_input(prompt)
if mode_input == 'y':
return (newconfig, path_changed)
elif mode_input == 'n':
abort = True
elif show_advanced and mode_input == 'a':
strategies = {
'MergeKeep': "(k)eep",
'MergeReplace': "(s)witch in",
'KillAppend': "(a)ppending"
}
unselected = [
v for k, v in list(strategies.items())
if k != merge_strategy
]
print(
"""New entries will just be appended to the config and
appear at the beginning of your ROS_PACKAGE_PATH. The merge strategy
decides how to deal with entries having a duplicate localname or path.
"(k)eep" means the existing entry will stay as it is, the new one will
be discarded. Useful for getting additional elements from other
workspaces without affecting your setup.
"(s)witch in" means that the new entry will replace the old in the
same position. Useful for upgrading/downgrading.
"switch (a)ppend" means that the existing entry will be removed, and
the new entry appended to the end of the list. This maintains order
of elements in the order they were given.
Switch append is the default.
""")
prompt = "Change Strategy %s: " % (", ".join(unselected))
mode_input = Ui.get_ui().get_input(prompt)
if mode_input == 's':
merge_strategy = 'MergeReplace'
elif mode_input == 'k':
merge_strategy = 'MergeKeep'
elif mode_input == 'a':
merge_strategy = 'KillAppend'
elif show_verbosity and mode_input == 'v':
extra_verbose = not extra_verbose
if abort:
print("No changes made.")
print('==========================================')
return (None, False)
def get_versioned_path_spec(self):
raise MultiProjectException(
"Cannot generate versioned outputs with non source types")
def __init__(self, path, local_name, properties=None):
self.path = path
if path is None:
raise MultiProjectException("Invalid empty path")
self.local_name = local_name
self.properties = properties
def get_workspace(argv, shell_path, config_filename=None, varname=None):
"""
If target option -t is given return value of that one. Else, if varname
is given and exists, considers that one, plus,
if config_filename is given, searches for a file named in config_filename
in 'shell_path' and ancestors.
In that case, if two solutions are found, asks the user.
:param shell_path: where to look for relevant config_filename
:param config_filename: optional, filename for files defining workspaces
:param varname: optional,
:returns: abspath if a .rosinstall was found, error and exist else.
"""
parser = OptionParser()
parser.add_option("-t",
"--target-workspace",
dest="workspace",
default=None,
help="which workspace to use",
action="store")
# suppress errors based on any other options this parser is agnostic about
argv2 = [
x for x in argv
if ((not x.startswith('-')) or x.startswith('--target-workspace=')
or x.startswith('-t') or x == '--target-workspace')
]
(options, args) = parser.parse_args(argv2)
if options.workspace is not None:
if (config_filename is not None and not os.path.isfile(
os.path.join(options.workspace, config_filename))):
raise MultiProjectException(
"%s has no workspace configuration file '%s'" %
(os.path.abspath(options.workspace), config_filename))
return os.path.abspath(options.workspace)
varname_path = None
if varname is not None and varname in os.environ:
# workspace could be relative, maybe confusing, but that's the users fault
varname_path = os.environ[varname]
if varname_path.strip() == '' or not os.path.isdir(varname_path):
varname_path = None
# use current dir
current_path = None
if config_filename is not None:
while shell_path is not None and not shell_path == os.path.dirname(
shell_path):
if os.path.exists(os.path.join(shell_path, config_filename)):
current_path = shell_path
break
shell_path = os.path.dirname(shell_path)
if current_path is not None and varname_path is not None and not samefile(
current_path, varname_path):
raise MultiProjectException(
"Ambiguous workspace: %s=%s, %s" %
(varname, varname_path, os.path.abspath(config_filename)))
if current_path is None and varname_path is None:
raise MultiProjectException("Command requires a target workspace.")
if current_path is not None:
return current_path
else:
return varname_path
def cmd_install_or_update(config,
backup_path=None,
mode='abort',
robust=False,
localnames=None,
num_threads=1,
verbose=False):
"""
performs many things, generally attempting to make
the local filesystem look like what the config specifies,
pulling from remote sources the most recent changes.
The command may have stdin user interaction (TODO abstract)
:param backup_path: if and where to backup trees before deleting them
:param robust: proceed to next element even when one element fails
:returns: True on Success
:raises MultiProjectException: on plenty of errors
"""
success = True
if not os.path.exists(config.get_base_path()):
os.mkdir(config.get_base_path())
# Prepare install operation check filesystem and ask user
preparation_reports = []
elements = select_elements(config, localnames)
for tree_el in elements:
abs_backup_path = None
if backup_path is not None:
abs_backup_path = os.path.join(config.get_base_path(), backup_path)
try:
preparation_report = tree_el.prepare_install(
backup_path=abs_backup_path, arg_mode=mode, robust=robust)
if preparation_report is not None:
if preparation_report.abort:
raise MultiProjectException(
"Aborting install because of %s" %
preparation_report.error)
if not preparation_report.skip:
preparation_reports.append(preparation_report)
else:
if preparation_report.error is not None:
print("Skipping install of %s because: %s" % (
preparation_report.config_element.get_local_name(),
preparation_report.error))
except MultiProjectException as exc:
fail_str = "Failed to install tree '%s'\n %s" % (
tree_el.get_path(), exc)
if robust:
success = False
print("Continuing despite %s" % fail_str)
else:
raise MultiProjectException(fail_str)
class Installer():
def __init__(self, report):
self.element = report.config_element
self.report = report
def do_work(self):
self.element.install(checkout=self.report.checkout,
backup=self.report.backup,
backup_path=self.report.backup_path,
verbose=self.report.verbose)
return {}
work = DistributedWork(len(preparation_reports), num_threads, silent=False)
for report in preparation_reports:
report.verbose = verbose
thread = Installer(report)
work.add_thread(thread)
try:
work.run()
except MultiProjectException as exc:
print("Exception caught during install: %s" % exc)
success = False
if not robust:
raise exc
return success
def cmd_update(self, target_path, argv, config=None):
parser = OptionParser(
usage="usage: %s update [localname]*" % self.progname,
formatter=IndentedHelpFormatterWithNL(),
description=__MULTIPRO_CMD_DICT__["update"] + """
This command calls the SCM provider to pull changes from remote to
your local filesystem. In case the url has changed, the command will
ask whether to delete or backup the folder.
Examples:
$ %(progname)s update -t ~/fuerte
$ %(progname)s update robot_model geometry
""" % {'progname': self.progname},
epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
parser.add_option(
"--delete-changed-uris",
dest="delete_changed",
default=False,
help="Delete the local copy of a directory before changing uri.",
action="store_true")
parser.add_option("--abort-changed-uris",
dest="abort_changed",
default=False,
help="Abort if changed uri detected",
action="store_true")
parser.add_option("--continue-on-error",
dest="robust",
default=False,
help="Continue despite checkout errors",
action="store_true")
parser.add_option(
"--backup-changed-uris",
dest="backup_changed",
default='',
help=
"backup the local copy of a directory before changing uri to this directory.",
action="store")
parser.add_option(
"-j",
"--parallel",
dest="jobs",
default=1,
help="How many parallel threads to use for installing",
action="store")
parser.add_option("-v",
"--verbose",
dest="verbose",
default=False,
help="Whether to print out more information",
action="store_true")
# -t option required here for help but used one layer above, see cli_common
parser.add_option("-t",
"--target-workspace",
dest="workspace",
default=None,
help="which workspace to use",
action="store")
(options, args) = parser.parse_args(argv)
if config is None:
config = multiproject_cmd.get_config(
target_path,
additional_uris=[],
config_filename=self.config_filename)
elif config.get_base_path() != target_path:
raise MultiProjectException("Config path does not match %s %s " %
(config.get_base_path(), target_path))
success = True
mode = _get_mode_from_options(parser, options)
if args == []:
# None means no filter, [] means filter all
args = None
if success:
install_success = multiproject_cmd.cmd_install_or_update(
config,
localnames=args,
backup_path=options.backup_changed,
mode=mode,
robust=options.robust,
num_threads=int(options.jobs),
verbose=options.verbose)
if install_success or options.robust:
return 0
return 1
def cmd_set(self, target_path, argv, config=None):
"""
command for modifying/adding a single entry
:param target_path: where to look for config
:param config: config to use instead of parsing file anew
"""
usage = (
"usage: %s set [localname] [SCM-URI]? [--(%ssvn|hg|git|bzr)] [--version=VERSION]]"
% (self.progname, 'detached|' if self.allow_other_element else ''))
parser = OptionParser(
usage=usage,
formatter=IndentedHelpFormatterWithNL(),
description=__MULTIPRO_CMD_DICT__["set"] + """
The command will infer whether you want to add or modify an entry. If
you modify, it will only change the details you provide, keeping
those you did not provide. if you only provide a uri, will use the
basename of it as localname unless such an element already exists.
The command only changes the configuration, to checkout or update
the element, run %(progname)s update afterwards.
Examples:
$ %(progname)s set robot_model --hg https://kforge.ros.org/robotmodel/robot_model
$ %(progname)s set robot_model --version-new robot_model-1.7.1
%(detached)s
""" % {
'progname':
self.progname,
'detached':
'$ %s set robot_model --detached' %
self.progname if self.allow_other_element else ''
},
epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
if self.allow_other_element:
parser.add_option(
"--detached",
dest="detach",
default=False,
help="make an entry unmanaged (default for new element)",
action="store_true")
parser.add_option("-v",
"--version-new",
dest="version",
default=None,
help="point SCM to this version",
action="store")
parser.add_option("--git",
dest="git",
default=False,
help="make an entry a git entry",
action="store_true")
parser.add_option("--svn",
dest="svn",
default=False,
help="make an entry a subversion entry",
action="store_true")
parser.add_option("--hg",
dest="hg",
default=False,
help="make an entry a mercurial entry",
action="store_true")
parser.add_option("--bzr",
dest="bzr",
default=False,
help="make an entry a bazaar entry",
action="store_true")
parser.add_option("-y",
"--confirm",
dest="confirm",
default='',
help="Do not ask for confirmation",
action="store_true")
# -t option required here for help but used one layer above, see cli_common
parser.add_option("-t",
"--target-workspace",
dest="workspace",
default=None,
help="which workspace to use",
action="store")
(options, args) = parser.parse_args(argv)
if not self.allow_other_element:
options.detach = False
if len(args) > 2:
print("Error: Too many arguments.")
print(parser.usage)
return -1
if config is None:
config = multiproject_cmd.get_config(
target_path,
additional_uris=[],
config_filename=self.config_filename)
elif config.get_base_path() != target_path:
raise MultiProjectException("Config path does not match %s %s " %
(config.get_base_path(), target_path))
scmtype = None
count_scms = 0
if options.git:
scmtype = 'git'
count_scms += 1
if options.svn:
scmtype = 'svn'
count_scms += 1
if options.hg:
scmtype = 'hg'
count_scms += 1
if options.bzr:
scmtype = 'bzr'
count_scms += 1
if options.detach:
count_scms += 1
if count_scms > 1:
parser.error(
"You cannot provide more than one scm provider option")
if len(args) == 0:
parser.error("Must provide a localname")
element = select_element(config.get_config_elements(), args[0])
uri = None
if len(args) == 2:
uri = args[1]
version = None
if options.version is not None:
version = options.version.strip("'\"")
# create spec object
if element is None:
if scmtype is None and not self.allow_other_element:
# for modification, not re-stating the scm type is
# okay, for new elements not
parser.error("You have to provide one scm provider option")
# asssume is insert, choose localname
localname = os.path.normpath(args[0])
rel_path = os.path.relpath(
os.path.realpath(localname),
os.path.realpath(config.get_base_path()))
if os.path.isabs(localname):
# use shorter localname for folders inside workspace
if not rel_path.startswith('..'):
localname = rel_path
else:
# got a relative path as localname, could point to a dir or be
# meant relative to workspace
if not samefile(os.getcwd(), config.get_base_path()):
if os.path.isdir(localname):
parser.error(
"Cannot decide which one you want to add:\n%s\n%s"
%
(os.path.abspath(localname),
os.path.join(config.get_base_path(), localname)))
if not rel_path.startswith('..'):
localname = rel_path
spec = PathSpec(local_name=localname,
uri=normalize_uri(uri, config.get_base_path()),
version=version,
scmtype=scmtype)
else:
# modify
old_spec = element.get_path_spec()
if options.detach:
spec = PathSpec(local_name=element.get_local_name())
else:
# '' evals to False, we do not want that
if version is None:
version = old_spec.get_version()
spec = PathSpec(local_name=element.get_local_name(),
uri=normalize_uri(uri or old_spec.get_uri(),
config.get_base_path()),
version=version,
scmtype=scmtype or old_spec.get_scmtype(),
path=old_spec.get_path())
if spec.get_legacy_yaml() == old_spec.get_legacy_yaml():
if not options.detach and spec.get_scmtype() is not None:
parser.error(
"Element %s already exists, did you mean --detached ?"
% spec)
parser.error("Element %s already exists" % spec)
(newconfig, path_changed) = prompt_merge(
target_path,
additional_uris=[],
additional_specs=[spec],
merge_strategy='MergeReplace',
confirmed=options.confirm,
confirm=not options.confirm,
show_verbosity=False,
show_advanced=False,
config_filename=self.config_filename,
config=config,
allow_other_element=self.allow_other_element)
if newconfig is not None:
print(
"Overwriting %s" %
os.path.join(newconfig.get_base_path(), self.config_filename))
shutil.move(
os.path.join(newconfig.get_base_path(),
self.config_filename), "%s.bak" %
os.path.join(newconfig.get_base_path(), self.config_filename))
self.config_generator(newconfig, self.config_filename)
if path_changed:
print(
"\nDo not forget to do ...\n$ source %s/setup.sh\n... in every open terminal."
% target_path)
if (spec.get_scmtype() is not None):
print(
"Config changed, remember to run '%s update %s' to update the folder from %s"
%
(self.progname, spec.get_local_name(), spec.get_scmtype()))
else:
print("New element %s could not be added, " % spec)
return 1
# auto-install not a good feature, maybe make an option
# for element in config.get_config_elements():
# if element.get_local_name() == spec.get_local_name():
# if element.is_vcs_element():
# element.install(checkout=not os.path.exists(os.path.join(config.get_base_path(), spec.get_local_name())))
# break
return 0
def cmd_merge(self, target_path, argv, config=None):
parser = OptionParser(
usage="usage: %s merge [URI] [OPTIONS]" % self.progname,
formatter=IndentedHelpFormatterWithNL(),
description=__MULTIPRO_CMD_DICT__["merge"] + """.
The command merges config with given other rosinstall element sets, from files or web uris.
The default workspace will be inferred from context, you can specify one using -t.
By default, when an element in an additional URI has the same
local-name as an existing element, the existing element will be
replaced. In order to ensure the ordering of elements is as
provided in the URI, use the option --merge-kill-append.
Examples:
$ %(prog)s merge someother.rosinstall
You can use '-' to pipe in input, as an example:
$ roslocate info robot_model | %(prog)s merge -
""" % {'prog': self.progname},
epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
# same options as for multiproject
parser.add_option(
"-a",
"--merge-kill-append",
dest="merge_kill_append",
default=False,
help="merge by deleting given entry and appending new one",
action="store_true")
parser.add_option(
"-k",
"--merge-keep",
dest="merge_keep",
default=False,
help="merge by keeping existing entry and discarding new one",
action="store_true")
parser.add_option(
"-r",
"--merge-replace",
dest="merge_replace",
default=True,
help=
"(default) merge by replacing given entry with new one maintaining ordering",
action="store_true")
parser.add_option(
"-y",
"--confirm-all",
dest="confirm_all",
default='',
help="do not ask for confirmation unless strictly necessary",
action="store_true")
# required here but used one layer above
parser.add_option("-t",
"--target-workspace",
dest="workspace",
default=None,
help="which workspace to use",
action="store")
(options, args) = parser.parse_args(argv)
if len(args) > 1:
print("Error: Too many arguments.")
print(parser.usage)
return -1
if len(args) == 0:
print("Error: Too few arguments.")
print(parser.usage)
return -1
config_uris = args
specs = []
if config_uris[0] == '-':
pipedata = "".join(sys.stdin.readlines())
try:
yamldicts = yaml.load(pipedata)
except yaml.YAMLError as e:
raise MultiProjectException("Invalid yaml format: \n%s \n%s" %
(pipedata, e))
if yamldicts is None:
parser.error("No Input read from stdin")
# cant have user interaction and piped input
options.confirm_all = True
specs.extend([get_path_spec_from_yaml(x) for x in yamldicts])
config_uris = []
merge_strategy = None
count_mergeoptions = 0
if options.merge_kill_append:
merge_strategy = 'KillAppend'
count_mergeoptions += 1
if options.merge_keep:
merge_strategy = 'MergeKeep'
count_mergeoptions += 1
if options.merge_replace:
merge_strategy = 'MergeReplace'
count_mergeoptions += 1
if count_mergeoptions > 1:
parser.error("You can only provide one merge-strategy")
# default option
if count_mergeoptions == 0:
merge_strategy = 'MergeReplace'
(newconfig,
_) = prompt_merge(target_path,
additional_uris=config_uris,
additional_specs=specs,
path_change_message="element order changed",
merge_strategy=merge_strategy,
confirmed=options.confirm_all,
config_filename=self.config_filename,
config=config,
allow_other_element=self.allow_other_element)
if newconfig is not None:
print(
"Config changed, maybe you need run %s update to update SCM entries."
% self.progname)
print(
"Overwriting %s" %
os.path.join(newconfig.get_base_path(), self.config_filename))
shutil.move(
os.path.join(newconfig.get_base_path(),
self.config_filename), "%s.bak" %
os.path.join(newconfig.get_base_path(), self.config_filename))
self.config_generator(newconfig, self.config_filename,
get_header(self.progname))
print("\nupdate complete.")
else:
print("Merge caused no change, no new elements found")
return 0
def cmd_info(self, target_path, argv, reverse=True, config=None):
parser = OptionParser(
usage="usage: %s info [localname]* [OPTIONS]" % self.progname,
formatter=IndentedHelpFormatterWithNL(),
description=__MULTIPRO_CMD_DICT__["info"] + """
The Status (S) column shows
x for missing
L for uncommited (local) changes
V for difference in version and/or remote URI
The 'Version-Spec' column shows what tag, branch or revision was given
in the .rosinstall file. The 'UID' column shows the unique ID of the
current (and specified) version. The 'URI' column shows the configured
URL of the repo.
If status is V, the difference between what was specified and what is
real is shown in the respective column. For SVN entries, the url is
split up according to standard layout (trunk/tags/branches).
When given one localname, just show the data of one element in list form.
This also has the generic properties element which is usually empty.
The --only option accepts keywords: %(opts)s
Examples:
$ %(prog)s info -t ~/ros/fuerte
$ %(prog)s info robot_model
$ %(prog)s info --yaml
$ %(prog)s info --only=path,cur_uri,cur_revision robot_model geometry
""" % {
'prog': self.progname,
'opts': ONLY_OPTION_VALID_ATTRS
},
epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
parser.add_option("--data-only",
dest="data_only",
default=False,
help="Does not provide explanations",
action="store_true")
parser.add_option(
"--only",
dest="only",
default=False,
help=
"Shows comma-separated lists of only given comma-separated attribute(s).",
action="store")
parser.add_option(
"--yaml",
dest="yaml",
default=False,
help="Shows only version of single entry. Intended for scripting.",
action="store_true")
parser.add_option("-u",
"--untracked",
dest="untracked",
default=False,
help="Also show untracked files as modifications",
action="store_true")
# -t option required here for help but used one layer above, see cli_common
parser.add_option("-t",
"--target-workspace",
dest="workspace",
default=None,
help="which workspace to use",
action="store")
(options, args) = parser.parse_args(argv)
if config is None:
config = multiproject_cmd.get_config(
target_path,
additional_uris=[],
config_filename=self.config_filename)
elif config.get_base_path() != target_path:
raise MultiProjectException("Config path does not match %s %s " %
(config.get_base_path(), target_path))
if args == []:
args = None
if options.only:
only_options = options.only.split(",")
if only_options == '':
parser.error('No valid options given')
lines = get_info_table_raw_csv(config,
properties=only_options,
localnames=args)
print('\n'.join(lines))
return 0
elif options.yaml:
source_aggregate = multiproject_cmd.cmd_snapshot(config,
localnames=args)
print(yaml.safe_dump(source_aggregate), end='')
return 0
# this call takes long, as it invokes scms.
outputs = multiproject_cmd.cmd_info(config,
localnames=args,
untracked=options.untracked)
if args and len(args) == 1:
# if only one element selected, print just one line
print(
get_info_list(config.get_base_path(), outputs[0],
options.data_only))
return 0
header = 'workspace: %s' % (target_path)
print(header)
table = get_info_table(config.get_base_path(),
outputs,
options.data_only,
reverse=reverse)
if table is not None and table != '':
print("\n%s" % table)
return 0