def get_from_archive(self, archive, src, dst, progress_cb):
# Note, that as we are in blocking mode, we can't easily fail on the
# actual get (that is 'cat'). Therefore check beforehand.
if not self.exists(archive):
raise RIARemoteError("archive {arc} does not exist."
"".format(arc=archive))
# TODO: We probably need to check exitcode on stderr (via marker). If
# archive or content is missing we will otherwise hang forever
# waiting for stdout to fill `size`.
cmd = '7z x -so {} {}\n'.format(sh_quote(str(archive)),
sh_quote(str(src)))
self.shell.stdin.write(cmd.encode())
self.shell.stdin.flush()
# TODO: - size needs double-check and some robustness
# - can we assume src to be a posixpath?
# - RF: Apart from the executed command this should be pretty much
# identical to self.get(), so move that code into a common
# function
from os.path import basename
size = self._get_download_size_from_key(basename(str(src)))
with open(dst, 'wb') as target_file:
bytes_received = 0
while bytes_received < size:
c = self.shell.stdout.read1(self.buffer_size)
if c:
bytes_received += len(c)
target_file.write(c)
progress_cb(bytes_received)
def symlink(src, dest):
log_cmd(["ln", "-s", src, dest])
try:
os.symlink(src, dest)
except OSError as e:
if hasattr(e, 'filename'):
sys.stderr.write("* ln: {}: {}\n".format(
sh_quote(e.filename), e.strerror))
else:
sys.stderr.write("* ln: {}: {}\n".format(
sh_quote(dest), e.strerror))
sys.exit(1)
def auto_done(args):
if args.longer_than:
print('export AUTO_NTFY_DONE_LONGER_THAN=-L{}'.format(
args.longer_than))
if args.unfocused_only:
print('export AUTO_NTFY_DONE_UNFOCUSED_ONLY=-b')
if args.shell == 'bash':
print('source {}'.format(sh_quote(scripts['bash-preexec.sh'])))
print('source {}'.format(sh_quote(scripts['auto-ntfy-done.sh'])))
print("# To use ntfy's shell integration, run "
"this and add it to your shell's rc file:")
print('# eval "$(ntfy shell-integration)"')
return None, None
def auto_done(args):
if args.longer_than:
print('export AUTO_NTFY_DONE_LONGER_THAN=-L{}'.format(
args.longer_than))
if args.unfocused_only:
print('export AUTO_NTFY_DONE_UNFOCUSED_ONLY=-b')
if args.shell == 'bash':
print('source {}'.format(sh_quote(scripts['bash-preexec.sh'])))
print('source {}'.format(sh_quote(scripts['auto-ntfy-done.sh'])))
print("# To use ntfy's shell integration, run "
"this and add it to your shell's rc file:")
print('# eval "$(ntfy shell-integration)"')
return None, None
def mkdir_p(dir):
log_cmd(["mkdir", "-p", dir])
# In 2.7, os.makedirs does not have the exist_ok parameter.
if os.path.isdir(dir):
return
try:
os.makedirs(dir)
except OSError as e:
if hasattr(e, 'filename'):
sys.stderr.write("* mkdir: {}: {}\n".format(
sh_quote(e.filename), e.strerror))
else:
sys.stderr.write("* mkdir: {}: {}\n".format(
sh_quote(dir), e.strerror))
sys.exit(1)
def chdir(dest):
log_cmd(["cd", dest])
try:
os.chdir(dest)
except OSError as e:
sys.stderr.write("* cd: {}: {}\n".format(sh_quote(dest), e.strerror))
sys.exit(1)
def in_archive(self, archive_path, file_path):
if not self.exists(archive_path):
return False
loc = str(file_path)
# query 7z for the specific object location, keeps the output
# lean, even for big archives
cmd = '7z l {} {}'.format(sh_quote(str(archive_path)), sh_quote(loc))
# Note: Currently relies on file_path not showing up in case of failure
# including non-existent archive. If need be could be more sophisticated
# and called with check=True + catch RemoteCommandFailedError
out = self._run(cmd, no_output=False, check=False)
return loc in out
def whereis(self, key):
dsobj_dir, archive_path, key_path = self._get_obj_location(key)
return str(key_path) if self._local_io() \
else '{}: {}:{}'.format(
self.storage_host,
self.remote_git_dir,
sh_quote(str(key_path)),
)
def touch(path):
log_cmd(["touch", path])
try:
os.utime(path, None)
except OSError as e:
sys.stderr.write("* touch: {}: {}\n".format(
sh_quote(path), e.strerror))
sys.exit(1)
def write_file(self, file_path, content, mode='w'):
if mode == 'w':
mode = ">"
elif mode == 'a':
mode = ">>"
else:
raise ValueError("Unknown mode '{}'".format(mode))
if not content.endswith('\n'):
content += '\n'
cmd = "printf '%s' {} {} {}".format(sh_quote(content), mode,
sh_quote(str(file_path)))
try:
self._run(cmd, check=True)
except RemoteCommandFailedError:
raise RIARemoteError("Could not write to {}".format(
str(file_path)))
def read_file(self, file_path):
cmd = "cat {}".format(sh_quote(str(file_path)))
try:
out = self._run(cmd, no_output=False, check=True)
except RemoteCommandFailedError:
raise RIARemoteError("Could not read {}".format(str(file_path)))
return out
def run(*argv):
log_cmd(argv)
try:
subprocess.check_call(argv)
except subprocess.CalledProcessError as e:
if e.returncode > 0:
sys.stderr.write("* {}: exit {}\n".format(
sh_quote(argv[0]), e.returncode))
else:
sys.stderr.write("* {}: signal {}\n".format(
sh_quote(argv[0]), -e.returncode))
sys.exit(1)
except OSError as e:
sys.stderr.write("* {}: {}\n".format(sh_quote(argv[0]), e.strerror))
sys.exit(1)
except Exception as e:
sys.stderr.write("* {}: {}\n".format(sh_quote(argv[0]), str(e)))
sys.exit(1)
def run_output(*argv):
sys.stderr.write("+ $(" + " ".join(sh_quote(arg) for arg in argv) + ") = ")
sys.stderr.flush()
raw_output = b''
try:
raw_output = subprocess.check_output(argv)
output = raw_output.decode(OUTPUT_ENCODING).strip()
if "\n" in output:
sys.stderr.write('"""\n' + output + '\n"""\n')
else:
sys.stderr.write(sh_quote(output) + "\n")
return output
except subprocess.CalledProcessError as e:
if e.returncode > 0:
sys.stderr.write("*\n* {}: exit {}\n".format(
sh_quote(argv[0]), e.returncode))
else:
sys.stderr.write("*\n* {}: signal {}\n".format(
sh_quote(argv[0]), -e.returncode))
sys.exit(1)
except OSError as e:
sys.stderr.write("*\n* {}: {}\n".format(sh_quote(argv[0]), e.strerror))
sys.exit(1)
except Exception as e:
sys.stderr.write("*\n* {}: decoding subprocess output: {}\n"
"* raw output: {!r}\n"
.format(sh_quote(argv[0]), str(e), raw_output))
sys.exit(1)
def get(self, src, dst, progress_cb):
# Note, that as we are in blocking mode, we can't easily fail on the
# actual get (that is 'cat').
# Therefore check beforehand.
if not self.exists(src):
raise RIARemoteError("annex object {src} does not exist."
"".format(src=src))
# TODO: see get_from_archive()
# TODO: Currently we will hang forever if the file isn't readable and
# it's supposed size is bigger than whatever cat spits out on
# stdout. This is because we don't notice that cat has exited
# non-zero. We could have end marker on stderr instead, but then
# we need to empty stderr beforehand to not act upon output from
# earlier calls. This is a problem with blocking reading, since we
# need to make sure there's actually something to read in any
# case.
cmd = 'cat {}'.format(sh_quote(str(src)))
self.shell.stdin.write(cmd.encode())
self.shell.stdin.write(b"\n")
self.shell.stdin.flush()
from os.path import basename
key = basename(str(src))
try:
size = self._get_download_size_from_key(key)
except RemoteError as e:
raise RemoteError("src: {}".format(str(src)) + str(e))
if size is None:
# rely on SCP for now
self.ssh.get(str(src), str(dst))
return
with open(dst, 'wb') as target_file:
bytes_received = 0
while bytes_received < size:
# TODO: some additional abortion criteria? check stderr in
# addition?
c = self.shell.stdout.read1(self.buffer_size)
# no idea yet, whether or not there's sth to gain by a
# sophisticated determination of how many bytes to read at once
# (like size - bytes_received)
if c:
bytes_received += len(c)
target_file.write(c)
progress_cb(bytes_received)
def whereis(self, key):
if isinstance(self.io, HTTPRemoteIO):
# display the URL for a request
# TODO: method of HTTPRemoteIO
return self.ria_store_url[4:] + "/annex/objects" + \
self.annex.dirhash(key) + "/" + key + "/" + key
dsobj_dir, archive_path, key_path = self._get_obj_location(key)
return str(key_path) if self._local_io() \
else '{}: {}:{}'.format(
self.storage_host,
self.remote_git_dir,
sh_quote(str(key_path)),
)
def generate_virt_install_cmd(vm, vm_defaults, extra_args=None):
"""generate vm xml using virt-install
:param vm: Element node representing vm
:param vm_defaults:
:returns: an array representing the virt-install command to import
the vm.
"""
if not extra_args:
etree.SubElement(vm, 'import')
extra_args = []
# etree.SubElement(vm, 'transient')
# etree.SubElement(vm, 'noautoconsole')
cmd_array = ['virt-install']
name = vm.find('name').text
if vm_defaults is not None:
vm_extend(vm, vm_defaults)
if vm.find("disk") is None:
# assume:
# 1. cwd mounted in the same location in docker
# 2. a qcow2 exist with the name of the vm
disk_element = etree.SubElement(vm, 'disk')
disk_element.text = os.path.join(os.getcwd(), name + ".qcow2")
for arg_i in list(vm):
cmd_array.append('--' + arg_i.tag)
val = None
if arg_i.attrib:
attrs = ",".join(
["{}={}".format(k, v) for k, v in arg_i.attrib.items()])
val = attrs
else:
val = arg_i.text
if val:
cmd_array.append(sh_quote(val))
cmd_array = cmd_array + extra_args
return cmd_array
def run(c, url, args, *backends):
file = c.urlparse(url, recurse=False)
name, ver = namever(file)
for backend in backends:
b = backend()
tgt = b._install_target(name, ver)
if tgt is None:
raise CompileError("Cannot find the target to run. Did you forget to\n\n quark install %s %s\n" % (b.argswitch, sh_quote(url)))
b.run(name, ver, args)
def exists(self, path):
try:
self._run('test -e {}'.format(sh_quote(str(path))), check=True)
return True
except RemoteCommandFailedError:
return False
def remove_dir(self, path):
self._run('rmdir {}'.format(sh_quote(str(path))))
def rename(self, src, dst):
self._run('mv {} {}'.format(sh_quote(str(src)), sh_quote(str(dst))))
def mkdir(self, path):
self._run('mkdir -p {}'.format(sh_quote(str(path))))
def _append_end_markers(self, cmd):
"""Append end markers to remote command"""
return cmd + " && printf '%s\\n' {} || printf '%s\\n' {}\n".format(
sh_quote(self.REMOTE_CMD_OK), sh_quote(self.REMOTE_CMD_FAIL))
def symlink(self, target, link_name):
self._run('ln -s {} {}'.format(sh_quote(str(target)),
sh_quote(str(link_name))))
def log_cmd(argv):
sys.stderr.write("+ " + " ".join(sh_quote(arg) for arg in argv) + "\n")
sys.stderr.flush()
def _parse_example(self, m, name, lineno):
"""
Given a regular expression match from `_EXAMPLE_RE` (`m`),
return a pair `(source, want)`, where `source` is the matched
example's source code (with prompts and indentation stripped);
and `want` is the example's expected output (with indentation
stripped).
`name` is the string's name, and `lineno` is the line number
where the example starts; both are used for error messages.
"""
# The regex matches both code examples and file
# constructions. Code examples are indented by `indent`.
if m.group("indent"):
# Get the example's indentation level.
indent = len(m.group("indent"))
# Divide source into lines; check that they're properly
# indented; and then strip their indentation & prompts.
lines = m.group("example").split("\n")[:-1]
source = ""
want = []
example_lineno = 0
# Parse line by line into separate examples
for l, line in enumerate(lines):
if not line.strip():
line = ""
else:
line = line[4:]
if line.startswith("$ "):
if self._IS_BLANK_OR_COMMENT(source):
pass
else:
# Extract options from the source.
options = self._find_options(source, name, lineno)
yield Example(
source,
"\n".join(want),
"",
lineno=lineno + example_lineno,
indent=indent,
options=options,
)
source = line[2:]
want = []
example_lineno = l
elif line.startswith("> "):
if want:
want.append(line)
else:
source = "{:}\n{:}".format(source, line[2:])
else:
want.append(line)
# Construct the last example.
# Extract options from the source.
options = self._find_options(source, name, lineno)
yield Example(
source,
"\n".join(want),
"",
lineno=lineno + example_lineno,
indent=indent,
options=options,
)
# File constructions, on the other hand, don't match that
# branch of the regular expression. Instead, they are have two
# other indentation levels, but `preindent` is only used to
# find the corresponding file name after it.
elif m.group("preindent"):
indent = len(m.group("fullindent"))
# Divide file into lines; check that they're properly
# indented; and then strip their indentation.
file_lines = m.group("content").split("\n")
self._check_prefix(file_lines, " " * indent, name, lineno)
file_content = "\n".join([fl[indent:] for fl in file_lines])
# The file name is just that, stripped.
filename = m.group("filename")
options = self._find_options(m.group("options"), name, lineno)
options[CREATE_FILE_BEFORE_TEST] = True
yield Example(
"cat {:s}".format(sh_quote(filename)),
file_content,
None,
lineno=lineno,
indent=indent,
options=options,
)
