def parse_param(input_stream, node, echars):
"""
Parse a parameter: may be quoted (in which case ends at ") else ends at echars
"""
skip_whitespace(input_stream)
if input_stream.peek() == '"':
input_stream.next()
# Skip the quote.
closing_quote = set(['"'])
parse_document(input_stream, node, closing_quote, True)
# Skip the '"'
c = input_stream.peek()
if c != '"':
raise utils.GiveUp('Quoted parameter did not end with %r: %s' %
('"', input_stream.report()))
input_stream.next()
skip_whitespace(input_stream)
c = input_stream.peek()
if (c in echars):
# Fine.
return
else:
input_stream.print_what_we_just_read()
raise utils.GiveUp(
"Quoted parameter ends with invalid character %r: "
"%s" % (c, input_stream.report()))
else:
parse_document(input_stream, node, echars, True)
def pull(self, repo, options, upstream=None, verbose=True):
"""
Will be called in the actual checkout's directory.
This runs Subversion's "update", but only if no merging will be
needed. That is, first it runs "svn status", and if any lines
contain a "C" (for Conflict) in columns 0, 1 or 6, then it will not
perform the update.
("svn help status" would call those columns 1, 2 and 7)
"""
if repo.branch:
raise utils.GiveUp("Subversion does not support branch"
" in 'pull' (branch='%s')"%repo.branch)
text = utils.get_cmd_data("svn status")
for line in text:
if 'C' in (line[0], line[1], line[6]):
raise utils.GiveUp("%s: 'svn status' says there is a Conflict,"
" refusing to pull:\n%s\nUse 'muddle merge'"
" if you want to merge"%(utils.indent(text,' ')))
starting_revno = self._just_revno()
utils.shell(["svn", "update"] + self._r_option(repo.revision), show_command=verbose)
# We could try parsing the output of 'svn update' instead, but this is
# simpler to do...
ending_revno = self._just_revno()
# Did we update anything?
return starting_revno != ending_revno
def clone_from_xml(self, node):
if (node.nodeType != node.ELEMENT_NODE or node.nodeName != "mknod"):
raise utils.GiveUp(
"Invalid outer element for %s user instruction - %s" %
("mknod", node))
result = MakeDeviceInstruction()
for c in node.childNodes:
if (c.nodeType == c.ELEMENT_NODE):
if (c.nodeName == "name"):
result.file_name = sanitise_filename(utils.text_in_node(c))
elif (c.nodeName == "uid"):
result.uid = utils.text_in_node(c)
elif (c.nodeName == "gid"):
result.gid = utils.text_in_node(c)
elif (c.nodeName == "type"):
result.type = utils.text_in_node(c)
elif (c.nodeName == "major"):
result.major = utils.text_in_node(c)
elif (c.nodeName == "minor"):
result.minor = utils.text_in_node(c)
elif (c.nodeName == "mode"):
result.mode = utils.text_in_node(c)
else:
raise utils.GiveUp(
"Invalid node in mknod instruction: %s" % (c.nodeName))
result.validate()
return result
def pull(self, repo, options, upstream=None, verbose=True):
"""
Pull changes, but don't do a merge.
Will be called in the actual checkout's directory.
"""
if repo.branch:
raise utils.GiveUp(
"Bazaar does not support branch (in the muddle sense)"
" in 'pull' (branch='%s')" % repo.branch)
rspec = self._r_option(repo.revision)
# Refuse to pull if there are any local changes
env = self._derive_env()
self._is_it_safe(env)
starting_revno = self._just_revno()
text = self._run1("bzr pull %s %s" %
(rspec, self._normalised_repo(repo.url)),
env=env,
verbose=verbose)
print text
if (text.startswith('No revisions to pull') # older versions of bzr
or text.startswith('No revisions or tags to pull') # bzr v2.6
) and repo.revision:
# Try going back to that particular revision.
#
# First we 'uncommit' to take our history back. The --force answers
# 'yes' to all questions (otherwise the user would be prompted as
# to whether they really wanted to do this operation
retcode, text = self._run2("bzr uncommit --force --quiet %s" %
rspec,
env=env,
verbose=verbose)
if retcode:
raise utils.GiveUp(
'Error uncommiting to revision %s (we already tried'
' pull)\nReturn code %d\n%s' % (rspec, retcode, text))
print text
# Then we need to 'revert' to undo any changes (since uncommit
# doesn't change our working set). The --no-backup stops us
# being left with copies of the changes in backup files (which
# is exactly what we don't want)
retcode, text = self._run2("bzr revert --no-backup %s" % rspec,
env=env,
verbose=verbose)
if retcode:
raise utils.GiveUp('Error reverting to revision %s (we already'
' uncommitted)\nReturn code %d\n%s' %
(rspec, retcode, text))
print text
ending_revno = self._just_revno()
# Did we update anything?
return starting_revno != ending_revno
def _decode_file_url(url):
result = urlparse.urlparse(url)
if result.scheme not in ('', 'file'):
raise utils.GiveUp("'%s' is not a valid 'file:' URL" % url)
if result.netloc:
raise utils.GiveUp("'%s' is not a valid 'file:' URL - wrong number"
" of '/' characters?" % url)
if result.params or result.query or result.fragment:
raise utils.GiveUp("'%s' is not a valid 'file:' URL - don't understand"
" params, query or fragment" % url)
return result.path
def merge(self, other_repo, options, verbose=True):
"""
Merge 'other_repo' into the local repository and working tree,
'bzr merge' will not (by default) merge if there are uncommitted changes
in the destination (i.e., local) tree. This is what we want.
Will be called in the actual checkout's directory.
"""
if other_repo.branch:
raise utils.GiveUp(
"Bazaar does not support branch (in the muddle sense)"
" in 'merge' (branch='%s')" % other_repo.branch)
# Refuse to pull if there are any local changes
env = self._derive_env()
self._is_it_safe(env)
rspec = self._r_option(other_repo.revision)
starting_revno = self._just_revno()
utils.shell("bzr merge %s %s" %
(rspec, self._normalised_repo(other_repo.url)),
env=env,
show_command=verbose)
ending_revno = self._just_revno()
# Did we update anything?
return starting_revno != ending_revno
def build_label(self, builder, label):
if (label.tag == utils.LabelTag.Deployed):
self.deploy(builder, label)
else:
raise utils.GiveUp("Attempt to build "
"unrecognised tools deployment label %s" %
(label))
def get(self, inOldValue, language):
if language == EnvLanguage.Value:
return self.get_value(inOldValue)
elif language == EnvLanguage.Sh:
return self.get_sh(inOldValue, True)
elif (language == EnvLanguage.C):
# As it says, the code wouldn't have worked anyway
# TODO: Work out what it should do
raise utils.GiveUp("attempt to get value '%s' for C, which is broken"%(inOldValue))
##return self.get_c(True)
elif language == EnvLanguage.Python:
return self.get_py(inOldValue)
else:
# What else can we do?
raise utils.GiveUp("attempt to get value '%s' for language %d"%(inOldValue,
language))
def apply_instructions(self, builder, label):
for role, domain in self.roles:
lbl = depend.Label(utils.LabelType.Package,
"*",
role,
"*",
domain=domain)
deploy_dir = builder.deploy_path(label)
instr_list = builder.load_instructions(lbl)
for (lbl, fn, instrs) in instr_list:
print "File deployment: Applying instructions for role %s, label %s .. " % (
role, lbl)
for instr in instrs:
# Obey this instruction.
iname = instr.outer_elem_name()
print 'Instruction:', iname
if iname in self.app_dict:
self.app_dict[iname].apply(builder, instr, role,
deploy_dir)
else:
raise utils.GiveUp(
"File deployments don't know about instruction %s"
% iname + " found in label %s (filename %s)" %
(lbl, fn))
def merge(self, other_repo, options, verbose=True):
"""
Merge 'other_repo' into the local repository and working tree,
Just copies everything again. This is an imperfect sort of "merge".
"""
if other_repo.revision and other_repo.revision != 'HEAD':
raise utils.GiveUp(
"File does not support the 'revision' argument to"
" 'merge' (revision='%s'" % other_repo.revision)
if other_repo.branch:
raise utils.GiveUp("File does not support the 'branch' argument to"
" 'merge' (branch='%s'" % other_repo.branch)
self.checkout(other_repo, os.curdir, options, verbose=verbose)
# See the comment in 'pull()' above
return True
def our_cmd(cmd_list, error_ok=True):
"""Command processing for calculating muddle version
"""
try:
p = subprocess.Popen(cmd_list, shell=False, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = p.communicate()
if p.returncode == 0:
return out.strip()
elif error_ok:
return ''
else:
raise utils.GiveUp("Problem determining muddle version: 'git' returned %s\n\n"
"$ %s\n"
"%s\n"%(p.returncode, ' '.join(cmd_list), out.strip()))
except OSError as e:
if e.errno == errno.ENOENT:
raise utils.GiveUp("Unable to determine 'muddle --version' - cannot find 'git'")
def _is_it_safe(self, env):
"""
No dentists here...
Raise an exception if there are (uncommitted) local changes.
"""
ok, cmd, txt = self._all_checked_in(env)
if not ok:
raise utils.GiveUp("There are uncommitted changes")
def revision_to_checkout(self, repo, co_leaf, options, force=False, before=None, verbose=False):
"""
Determine a revision id for this checkout, usable to check it out again.
Uses 'svnversion', which I believe is installed as standard with 'svn'
For the moment, at lease, the 'force' argument is ignored (so the
working copy must be be equivalent to the repository).
"""
if before:
raise utils.GiveUp('%s: "before" argument not currently supported'%co_leaf)
revision = utils.get_cmd_data('svnversion', show_command=verbose)
revision = revision.strip()
if all([x.isdigit() for x in revision]):
return revision
else:
raise utils.GiveUp("%s: 'svnversion' reports checkout has revision"
" '%s'"%(co_leaf, revision))
def checkout(self, repo, co_leaf, options, verbose=True):
"""
Clone a given checkout.
Will be called in the parent directory of the checkout.
Expected to create a directory called <co_leaf> therein.
"""
if repo.revision and repo.revision != 'HEAD':
raise utils.GiveUp(
"File does not support the 'revision' argument to"
" 'checkout' (revision='%s'" % repo.revision)
if repo.branch:
raise utils.GiveUp("File does not support the 'branch' argument to"
" 'checkout' (branch='%s'" % repo.branch)
parsed = urlparse.urlparse(repo.url)
source_path = parsed.path
utils.recursively_copy(source_path, co_leaf, preserve=True)
def dependency_sort(self):
"""
Sort self.vars.items() in as close to dependency order as you can.
"""
# deps maps environment variables to a set of the variables they
# (directly) depend on.
deps = { }
remain = set()
for (k,v) in self.vars.items():
deps[k] = v.dependencies()
remain.add(k)
done = set()
out_list = [ ]
while len(remain) > 0:
# Take out anything we can
new_remain = set()
did_something = False
for k in remain:
can_issue = True
for cur_dep in deps[k]:
if cur_dep not in done:
# Curses: a dependency not in done.
can_issue = False
break
if (can_issue):
out_list.append(k)
done.add(k)
did_something = True
else:
# Can't issue
new_remain.add(k)
remain = new_remain
# If we didn't do anything, we've reached the end
# of the line.
if (not did_something):
raise utils.GiveUp("Cannot produce a consistent environment ordering:\n" +
("Issued: %s\n"%(" ".join(map(str, out_list)))) +
("Remain: %s\n"%utils.print_string_set(remain)) +
("Deps: %s\n"%(print_deps(deps))))
# Form the value list ..
rv = [ ]
for k in out_list:
rv.append((k, self.vars[k]))
return rv
def build_label(self, builder, label):
if (label.type == utils.LabelType.Deployment
and (label.tag == utils.LabelTag.Clean
or label.tag == utils.LabelTag.DistClean)):
deploy_path = builder.deploy_path(label)
print "> Remove %s" % deploy_path
utils.recursively_remove(deploy_path)
builder.kill_label(label.copy_with_tag(utils.LabelTag.Deployed))
else:
raise utils.GiveUp("Attempt to invoke CleanDeploymentBuilder on "
"unrecognised label %s" % label)
def pull(self, repo, options, upstream=None, verbose=True):
"""
Will be called in the actual checkout's directory.
Just copies everything again.
"""
if repo.revision and repo.revision != 'HEAD':
raise utils.GiveUp(
"File does not support the 'revision' argument to"
" 'pull' (revision='%s'" % repo.revision)
if repo.branch:
raise utils.GiveUp("File does not support the 'branch' argument to"
" 'pull' (branch='%s'" % repo.branch)
self.checkout(repo, os.curdir, options, verbose=verbose)
# Ideally, we would determine whether the directory had changed by
# using hashlib to do a recursive MD5 or SHA1 sum of the files in it.
# However, since I don't know that anyone is *using* this VCS method,
# it hardly seems worth the effort. So we'll lie.
return True
def lookup_command(command_name, args, cmd_dict, subcmd_dict):
"""
Look the command up, and return an instance of it and any remaining args
"""
try:
command_class = cmd_dict[command_name]
except KeyError:
raise utils.GiveUp("There is no muddle command '%s'"%command_name)
if command_class is None:
try:
subcommand_name = args[0]
except IndexError:
raise utils.GiveUp("Command '%s' needs a subcommand"%command_name)
args = args[1:]
try:
command_class = subcmd_dict[command_name][subcommand_name]
except KeyError:
raise utils.GiveUp("There is no muddle command"
" '%s %s'"%(command_name, subcommand_name))
return command_class(), args
def fnval(self, xml_doc, env, output_list):
if (len(self.params) != 1):
raise utils.GiveUp("val() must have exactly one parameter")
key_name = self.params[0].eval_str(xml_doc, env)
key_name = key_name.strip()
if (key_name is None):
res = ""
else:
res = query_string_value(xml_doc, env, key_name)
if (res is None):
raise utils.GiveUp(
"Attempt to substitute key '%s' which does not exist." %
key_name)
if (g_trace_parser):
print "node.fnval(%s -> %s) = %s" % (self.params[0], key_name, res)
output_list.append(res)
def _revision_id(self, env, revspec):
"""Find the revision id for revision 'revspec'
"""
cmd = "bzr log -l 1 -r '%s' --long --show-ids" % revspec
retcode, text = self._run2(cmd, env=env)
if retcode != 0:
raise utils.GiveUp("'%s' failed with return code %d\n%s" %
(cmd, retcode, text))
# Let's look for the revision-id field therein
lines = text.split('\n')
for line in lines:
line = line.strip()
parts = line.split(':')
if parts[0] == 'revision-id':
revision = ':'.join(
parts[1:]) # although I hope there aren't internal colons!
return revision.strip()
raise utils.GiveUp("'%s' did not return text containing 'revision-id:'"
"\n%s" % (cmd, text))
def ensure_dirs(self, builder, label):
inv = builder
tmp = Label(utils.LabelType.Checkout,
self.co_name,
domain=label.domain)
if not os.path.exists(inv.checkout_path(tmp)):
raise utils.GiveUp("Path for checkout %s does not exist." %
self.co_name)
utils.ensure_dir(inv.package_install_path(label))
utils.ensure_dir(inv.package_obj_path(label))
def ensure_dirs(self, builder, label):
inv = builder
# TODO: Does the following check one dir and then make another???
tmp = Label(utils.LabelType.Checkout,
self.co_name,
domain=label.domain)
if not os.path.exists(inv.checkout_path(tmp)):
raise utils.GiveUp("Path for checkout %s does not exist." %
self.co_name)
utils.ensure_dir(os.path.join(inv.package_obj_path(label), "obj"))
def checkout(self, repo, co_leaf, options, verbose=True):
"""
Clone a given checkout.
Will be called in the parent directory of the checkout.
Expected to create a directory called <co_leaf> therein.
"""
if repo.branch:
raise utils.GiveUp("Subversion does not support branch"
" in 'checkout' (branch='%s')"%repo.branch)
utils.shell(["svn", "checkout"] + self._r_option(repo.revision) +
[repo.url, co_leaf], show_command=verbose)
def __init__(self, inv, checkout_name, repo, rev, rel, checkout_dir):
VersionControlHandler.__init__(self, inv, checkout_name, repo, rev,
rel, checkout_dir)
sp = conventional_repo_url(repo, rel)
if sp is None:
raise utils.GiveUp(
"Cannot extract repository URL from %s, checkout %s" %
(repo, rel))
(self.url, r) = sp
parsed = urlparse.urlparse(self.url)
self.filename = os.path.split(parsed.path)[-1]
self.checkout_path = self.get_checkout_path(self.checkout_name)
def clone_from_xml(self, xmlNode):
"""
Clone a filespec from some XML like::
<filespec>
<root>..</root>
<spec> ..</spec>
<all-under />
<all-regex />
</filespec>
"""
if (xmlNode.nodeName != "filespec"):
raise utils.GiveUp(
"Filespec xml node is called %s , not filespec." %
(xmlNode.nodeName))
new_root = None
new_spec = None
new_all_under = False
new_all_regex = False
for c in xmlNode.childNodes:
if (c.nodeType == c.ELEMENT_NODE):
if (c.nodeName == "root"):
new_root = utils.text_in_node(c)
elif (c.nodeName == "spec"):
new_spec = utils.text_in_node(c)
elif (c.nodeName == "all-under"):
new_all_under = True
elif (c.nodeName == "all-regex"):
new_all_regex = True
else:
raise utils.GiveUp("Unknown element %s in filespec" %
(c.nodeName))
return FileSpec(new_root, new_spec, new_all_under, new_all_regex)
def normalise(self):
"""
Normalise the hierarchy into one with a single root.
We do this by taking each root in turn and removing a component, creating
a directory in the process. If the resulting root is in roots, we add it
to the children of that root and eliminate it from the map.
Iterate until there is only one root left.
"""
while len(self.roots) > 1:
# Pick a key ..
did_something = False
new_roots = {}
for (k, v) in self.roots.items():
#print "Normalise %s => "%k
if (k != "/"):
# It can be shortened ..
(dir, name) = os.path.split(k)
did_something = True
#print "=> dir = %s"%dir
if (dir not in self.map):
# Curses. Don't already have that directory -
# must build it.
new_dir = File()
new_dir.mode = 0755 | File.S_DIR
new_dir.name = dir
new_dir.children.append(v)
# The directory wasn't present, so must be
# a new root .
new_roots[dir] = new_dir
self.map[dir] = new_dir
else:
new_dir = self.map[dir]
new_dir.children.append(v)
else:
new_roots[k] = v
self.roots = new_roots
if (not did_something):
raise utils.GiveUp("Failed to normalise a hierarchy -"
" circular roots?: %s" % self)
def deployment_rule_from_name(builder, name):
"""
Return the rule for target label "deployment:<name>{}/deployed".
Raises an exception if there is more than one such rule.
"""
rules = builder.ruleset.rules_for_target(depend.Label(
utils.LabelType.Deployment, name, None, utils.LabelTag.Deployed),
useTags=True,
useMatch=False)
if (len(rules) != 1):
raise utils.GiveUp(
"Attempt to retrieve rule for deployment %s:" % name +
" returned list had length %d ,not 1.. " % len(rules))
for r in rules:
return r
def val(self, xml_doc, env, output_list):
key_name = self.expr.eval_str(xml_doc, env)
key_name = key_name.strip()
if (key_name is None):
res = ""
else:
res = query_string_value(xml_doc, env, key_name)
if (res is None):
raise utils.GiveUp(
"Attempt to substitute key '%s' which does not exist." %
key_name)
if (g_trace_parser):
print "node.val(%s -> %s) = %s" % (self.expr, key_name, res)
output_list.append(res)
def build_label(self, builder, label):
"""
Performs the actual build.
We actually do need to copy all files from install/ (where unprivileged
processes can modify them) to deploy/ (where they can't).
Then we apply instructions to deploy.
"""
if label.tag == utils.LabelTag.Deployed:
self.deploy(builder, label)
elif label.tag == utils.LabelTag.InstructionsApplied:
self.apply_instructions(builder, label)
else:
raise utils.GiveUp(
"Attempt to build a deployment with an unexpected tag in label %s"
% (label))
def validate(self):
if (self.file_name is None):
raise utils.GiveUp("Invalid mknod node - no file name")
if (self.uid is None):
raise utils.GiveUp("Invalid mknod node - no uid")
if (self.gid is None):
raise utils.GiveUp("Invalid mknod node - no gid")
if (self.type is None):
raise utils.GiveUp(
"Invalid mknod node - no device type (block or char)")
if (self.major is None):
raise utils.GiveUp("Invalid mknod node - no major number")
if (self.minor is None):
raise utils.GiveUp("Invalid mknod node - no minor number")
if (self.mode is None):
raise utils.GiveUp("Invalid mknod node - no mode")
