def to_curl(self):
method = self.method.upper()
if self.cookies:
try:
cookies = SmartCookie(self.cookies)
except Exception:
pass
else:
# The Cookie header is already yanked out of the headers dict
# inside `to_python` so we can just safely re-set it.
self.headers['Cookie'] = ';'.join(c.output(attrs=[], header='') for c in cookies.values()).strip()
bits = []
if method != 'GET':
bits.append('-X' + method)
data = self.data
if isinstance(data, dict):
data = urlencode(format_body(data))
if isinstance(data, basestring):
bits.append('--data ' + quote(data))
bits.append(quote(self.full_url))
for header in self.headers.iteritems():
bits.append('-H ' + quote('%s: %s' % header))
if 'gzip' in self.headers.get('Accept-Encoding', ''):
bits.append('--compressed')
return 'curl ' + ' '.join(bits)
def convert_mp3_to_wav(filename, sample_frequency):
ext = filename[-4:]
if(ext != '.mp3'):
return
files = filename.split('/')
orig_filename = files[-1][0:-4]
new_path = ''
if(filename[0] == '/'):
new_path = '/'
for i in xrange(len(files) - 1):
new_path += files[i] + '/'
tmp_path = new_path + 'tmp'
new_path += 'wave'
if not os.path.exists(new_path):
os.makedirs(new_path)
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
filename_tmp = tmp_path + '/' + orig_filename + '.mp3'
new_name = new_path + '/' + orig_filename + '.wav'
sample_freq_str = "{0:.1f}".format(float(sample_frequency) / 1000.0)
cmd = 'lame -a -m m {0} {1}'.format(quote(filename),
quote(filename_tmp))
os.system(cmd)
cmd = 'lame --decode {0} {1} --resample {2}'.format(
quote(filename_tmp),
quote(new_name),
sample_freq_str
)
os.system(cmd)
return new_name
def create_virtualenv(directory, system_site_packages=False, venv_python=None,
use_sudo=False, user=None, clear=False, prompt=None,
virtualenv_cmd='virtualenv'):
"""
Create a Python `virtual environment`_.
::
import fabtools
fabtools.python.create_virtualenv('/path/to/venv')
.. _virtual environment: http://www.virtualenv.org/
"""
options = ['--quiet']
if system_site_packages:
options.append('--system-site-packages')
if venv_python:
options.append('--python=%s' % quote(venv_python))
if clear:
options.append('--clear')
if prompt:
options.append('--prompt=%s' % quote(prompt))
options = ' '.join(options)
directory = quote(directory)
command = '%(virtualenv_cmd)s %(options)s %(directory)s' % locals()
if use_sudo:
sudo(command, user=user)
else:
run(command)
def __str__(self):
ret = ''
for attr in 'cmd', 'ret_code', 'out', 'err':
value = getattr(self, attr, None)
if value is not None and str(value).strip():
mesg = ''
if attr == 'cmd' and self.cmd_kwargs.get('stdin_file_paths'):
mesg += 'cat'
for file_path in self.cmd_kwargs.get('stdin_file_paths'):
mesg += ' ' + quote(file_path)
mesg += ' | '
if attr == 'cmd' and isinstance(value, list):
mesg += ' '.join(quote(item) for item in value)
else:
mesg = str(value).strip()
if attr == 'cmd' and self.cmd_kwargs.get('stdin_str'):
mesg += ' <<<%s' % quote(self.cmd_kwargs.get('stdin_str'))
if len(mesg.splitlines()) > 1:
fmt = self.JOB_LOG_FMT_M
else:
fmt = self.JOB_LOG_FMT_1
if not mesg.endswith('\n'):
mesg += '\n'
ret += fmt % {
'cmd_key': self.cmd_key,
'attr': attr,
'mesg': mesg}
return ret.rstrip()
def put_file(self, in_path, out_path):
""" transfer a file from local to remote """
vvv("PUT %s TO %s" % (in_path, out_path), host=self.host)
if not os.path.exists(in_path):
raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path)
cmd = self._password_cmd()
host = self.host
if self.ipv6:
host = "[%s]" % host
if C.DEFAULT_SCP_IF_SSH:
cmd += ["scp"] + self.common_args
cmd += [in_path, host + ":" + pipes.quote(out_path)]
indata = None
else:
cmd += ["sftp"] + self.common_args + [host]
indata = "put %s %s\n" % (pipes.quote(in_path), pipes.quote(out_path))
(p, stdin) = self._run(cmd, indata)
self._send_password()
(returncode, stdout, stderr) = self._communicate(p, stdin, indata)
if returncode != 0:
raise errors.AnsibleError("failed to transfer file to %s:\n%s\n%s" % (out_path, stdout, stderr))
def setUp(self):
"""
Create a temporary working directory and a virtual environment where
pip-accel can be tested in isolation (starting with an empty download
cache, source index and binary index and no installed modules) and make
sure pip and pip-accel use the directory. Also creates the directories
for the download cache, the source index and the binary index (normally
this is done from pip_accel.main).
"""
coloredlogs.install(level=logging.DEBUG)
# Create a temporary working directory.
self.working_directory = tempfile.mkdtemp()
self.download_cache = os.path.join(self.working_directory, 'download-cache')
# Create a temporary build directory.
self.build_directory = os.path.join(self.working_directory, 'build')
# Create a temporary virtual environment.
self.virtual_environment = os.path.join(self.working_directory, 'environment')
python = 'python%i.%i' % (sys.version_info[0], sys.version_info[1])
assert os.system('virtualenv --python=%s %s' % (pipes.quote(python), pipes.quote(self.virtual_environment))) == 0
# Make sure pip-accel uses the pip in the temporary virtual environment.
os.environ['PATH'] = '%s:%s' % (os.path.join(self.virtual_environment, 'bin'), os.environ['PATH'])
os.environ['VIRTUAL_ENV'] = self.virtual_environment
# Make pip and pip-accel use the temporary working directory.
os.environ['PIP_DOWNLOAD_CACHE'] = self.download_cache
os.environ['PIP_ACCEL_CACHE'] = self.working_directory
def setup_ssh_access(self):
"""
Update ``~/.ssh/config`` to make it easy to connect to the container
over SSH_ from the host system. This generates a host definition to
include in the SSH client configuration file and uses update-dotdee_ to
merge the generated host definition with the user's existing SSH client
configuration file.
.. _update-dotdee: https://pypi.python.org/pypi/update-dotdee
"""
self.logger.verbose("Configuring SSH access ..")
self.update_dotdee.create_directory()
with open(self.ssh_config_file, 'w') as handle:
handle.write(textwrap.dedent("""
Host {alias}
Hostname {address}
Port {port}
User root
IdentityFile {key}
StrictHostKeyChecking no
UserKnownHostsFile /dev/null
""".format(alias=self.ssh_alias,
address=self.ssh_endpoint[0],
port=self.ssh_endpoint[1],
key=PRIVATE_SSH_KEY,
redock=pipes.quote(os.path.abspath(sys.argv[0])),
container=pipes.quote(self.image.name))))
self.update_dotdee.update_file()
self.logger.info("Successfully configured SSH access. Use this command: ssh %s", self.ssh_alias)
def branch_package(pkgname, requested_branches, existing_branches):
'''Create all the branches that are listed in the pkgdb for a package.
:arg pkgname: The package to create branches for
:arg requested_branches: The branches to creates
:arg existing_branches: A list of existing local branches
'''
if VERBOSE:
print 'Fixing package %s for branches %s' % (pkgname, requested_branches)
# Create the devel branch if necessary
exists = os.path.exists(os.path.join(GIT_FOLDER, '%s.git' % pkgname))
if not exists or 'master' not in existing_branches:
emails = PKG_OWNER_EMAILS.replace("$PACKAGE", pkgname)
_invoke(SETUP_PACKAGE, ["--pkg-owner-emails", pipes.quote(emails),
"--email-domain", pipes.quote(EMAIL_DOMAIN),
"--default-branch-author", pipes.quote(DEFAULT_BRANCH_AUTHOR),
pkgname])
if 'master' in requested_branches:
requested_branches.remove('master') # SETUP_PACKAGE creates master
# Create all the required branches for the package
# Use the translated branch name until pkgdb falls inline
for branch in requested_branches:
_create_branch(pkgname, branch, existing_branches)
def install_packages(env_dir, opt):
"""
Install node.js packages via npm
"""
if is_windows_nt:
return install_packages_win(env_dir, opt)
logger.info(' * Install node.js packages ... ',
extra=dict(continued=True))
packages = [package.strip() for package in
open(opt.requirements).readlines()]
activate_path = join(env_dir, 'bin', 'activate')
real_npm_ver = opt.npm if opt.npm.count(".") == 2 else opt.npm + ".0"
if opt.npm == "latest" or real_npm_ver >= "1.0.0":
cmd = '. ' + pipes.quote(activate_path) + \
' && npm install -g %(pack)s'
else:
cmd = '. ' + pipes.quote(activate_path) + \
' && npm install %(pack)s' + \
' && npm activate %(pack)s'
for package in packages:
callit(cmd=[cmd % {"pack": package}],
show_stdout=opt.verbose, in_shell=True)
logger.info('done.')
def generate_date_fixer(r_export_txt_xls, input_dir, output_dir):
tables = complex_schema.read_tables(r_export_txt_xls)
for table in tables:
date_fields = [
field.name
for field in table.fields
if field.type == 'date'
]
if date_fields:
input_file = '{}.csv'.format(
os.path.normpath(os.path.join(input_dir, table.name))
)
output_file = '{}.csv'.format(
os.path.normpath(os.path.join(output_dir, table.name))
)
if os.path.exists(input_file):
command = 'drop-invalid-dates {} < {} > {}'.format(
quote(','.join(date_fields)),
quote(input_file),
quote(output_file),
)
yield command
def scp_from_remote(host, port, username, password, remote_path, local_path,
limit="", log_filename=None, timeout=600, interface=None):
"""
Copy files from a remote host (guest).
:param host: Hostname or IP address
:param username: Username (if required)
:param password: Password (if required)
:param local_path: Path on the local machine where we are copying from
:param remote_path: Path on the remote machine where we are copying to
:param limit: Speed limit of file transfer.
:param log_filename: If specified, log all output to this file
:param timeout: The time duration (in seconds) to wait for the transfer
to complete.
:interface: The interface the neighbours attach to (only use when
using ipv6 linklocal address.)
:raise: Whatever remote_scp() raises
"""
if (limit):
limit = "-l %s" % (limit)
if host and host.lower().startswith("fe80"):
if not interface:
raise SCPError("When using ipv6 linklocal address must assign, ",
"the interface the neighbour attache")
host = "%s%%%s" % (host, interface)
command = ("scp -v -o UserKnownHostsFile=/dev/null "
"-o StrictHostKeyChecking=no "
"-o PreferredAuthentications=password -r %s "
"-P %s %s@\[%s\]:%s %s" %
(limit, port, username, host, pipes.quote(remote_path), pipes.quote(local_path)))
password_list = []
password_list.append(password)
remote_scp(command, password_list, log_filename, timeout)
def __init__(self, url):
self.url = url
self.cmd = ['ssh', '-T'] # no pseudo-tty
if not url.path.startswith('/'):
raise urllib2.URLError("Bad path in URL %r" % url.geturl())
# pipes.quote is used because ssh always goes through a login shell.
# The only exception is for redirecting ports, which can't be used
# to access a domain socket.
if url.path.startswith("/~/"):
clean_path = "~/" + pipes.quote(url.path[3:])
else:
clean_path = pipes.quote(url.path)
ssh_netloc = ''.join((url.username or '', '@' if url.username else '', url.hostname))
if url.port:
reconstructed_netloc = '%s:%d' % (ssh_netloc, url.port)
self.cmd.extend(["-p", str(url.port)])
else:
reconstructed_netloc = ssh_netloc
if reconstructed_netloc != url.netloc:
raise urllib2.URLError("Bad location in URL %r (expected %r)" % (url.geturl(), reconstructed_netloc))
self.cmd.extend(["--", ssh_netloc])
#self.cmd.extend(["/bin/nc", "-U", "--", clean_path])
self.cmd.extend(["socat", "STDIO", "UNIX-CONNECT:" + clean_path])
def create_recording(qemu_path, qcow, snapshot, command, copy_directory, recording_path, isoname=None, rr=False):
DEVNULL = open(os.devnull, "w")
recording_path = realpath(recording_path)
if not isoname: isoname = copy_directory + '.iso'
with TempDir() as tempdir, Qemu(qemu_path, qcow, snapshot, tempdir, rr=rr) as qemu:
if os.listdir(copy_directory):
progress("Creating ISO {}...".format(isoname))
make_iso(copy_directory, isoname)
progress("Inserting CD...")
qemu.run_monitor("change ide1-cd0 \"{}\"".format(isoname))
qemu.run_console("mkdir -p {}".format(pipes.quote(copy_directory)))
# Make sure cdrom didn't automount
# Make sure guest path mirrors host path
qemu.run_console("while ! mount /dev/cdrom {}; ".format(pipes.quote(copy_directory)) +
"do sleep 0.3; umount /dev/cdrom; done")
# Important that we type command into console before recording starts and only
# hit enter once we've started the recording.
progress("Running command inside guest.")
qemu.type_console(subprocess32.list2cmdline(command))
# start PANDA recording
qemu.run_monitor("begin_record \"{}\"".format(recording_path))
qemu.run_console(timeout=1200)
# end PANDA recording
progress("Ending recording...")
qemu.run_monitor("end_record")
DEVNULL.close()
def db_dump(module, target, target_opts="",
db=None,
user=None,
password=None,
host=None,
port=None,
**kw):
flags = login_flags(db, host, port, user, db_prefix=False)
cmd = module.get_bin_path('pg_dump', True)
comp_prog_path = None
if os.path.splitext(target)[-1] == '.tar':
flags.append(' --format=t')
if os.path.splitext(target)[-1] == '.gz':
if module.get_bin_path('pigz'):
comp_prog_path = module.get_bin_path('pigz', True)
else:
comp_prog_path = module.get_bin_path('gzip', True)
elif os.path.splitext(target)[-1] == '.bz2':
comp_prog_path = module.get_bin_path('bzip2', True)
elif os.path.splitext(target)[-1] == '.xz':
comp_prog_path = module.get_bin_path('xz', True)
cmd += "".join(flags)
if target_opts:
cmd += " {0} ".format(target_opts)
if comp_prog_path:
cmd = '{0}|{1} > {2}'.format(cmd, comp_prog_path, pipes.quote(target))
else:
cmd = '{0} > {1}'.format(cmd, pipes.quote(target))
return do_with_password(module, cmd, password)
def setup_auto_startup_initd_linux(command, cwd, user=None, name='swarming'):
"""Uses init.d to start the bot automatically."""
if not user:
user = getpass.getuser()
logging.info(
'setup_auto_startup_initd_linux(%s, %s, %s, %s)',
command, cwd, user, name)
if not os.path.isabs(cwd):
raise ValueError('Refusing relative path')
script = _generate_initd(command, cwd, user)
filepath = pipes.quote(os.path.join('/etc/init.d', name))
with tempfile.NamedTemporaryFile() as f:
if not _write(f.name, script):
return False
# Need to do 3 things as sudo. Do it all at once to enable a single sudo
# request.
# TODO(maruel): Likely not the sanest thing, reevaluate.
cmd = [
'sudo', '/bin/sh', '-c',
"cp %s %s && chmod 0755 %s && update-rc.d %s defaults" % (
pipes.quote(f.name), filepath, filepath, name)
]
subprocess.check_call(cmd)
print('To remove, use:')
print(' sudo update-rc.d -f %s remove' % name)
print(' sudo rm %s' % filepath)
return True
def slave_command(self, extra_args=None):
"""Create the `wmt-slave` command.
Parameters
----------
extra_args : str, optional
Additional arguments.
Returns
-------
str
The slave command to execute.
"""
import shlex
from pipes import quote
wmt_slave = os.path.join(sys.prefix, 'bin', 'wmt-slave')
command = [wmt_slave, quote(self.sim_id)] + self._extra_args
if self.server_url:
command += ['--server-url={}'.format(self.server_url)]
if extra_args:
if isinstance(extra_args, str):
extra_args = shlex.split(extra_args)
command += [quote(arg) for arg in extra_args]
return ' '.join(command)
def transcode_commands(output_format, resample, needed_sample_rate, flac_file, transcode_file):
'''
Return a list of transcode steps (one command per list element),
which can be used to create a transcode pipeline for flac_file ->
transcode_file using the specified output_format, plus any
resampling, if needed.
'''
if resample:
flac_decoder = 'sox %(FLAC)s -G -b 16 -t wav - rate -v -L %(SAMPLERATE)s dither'
else:
flac_decoder = 'flac -dcs -- %(FLAC)s'
lame_encoder = 'lame -S %(OPTS)s - %(FILE)s'
flac_encoder = 'flac %(OPTS)s -o %(FILE)s -'
transcoding_steps = [flac_decoder]
if encoders[output_format]['enc'] == 'lame':
transcoding_steps.append(lame_encoder)
elif encoders[output_format]['enc'] == 'flac':
transcoding_steps.append(flac_encoder)
transcode_args = {
'FLAC' : pipes.quote(flac_file),
'FILE' : pipes.quote(transcode_file),
'OPTS' : encoders[output_format]['opts'],
'SAMPLERATE' : needed_sample_rate,
}
if output_format == 'FLAC' and resample:
commands = ['sox %(FLAC)s -G -b 16 %(FILE)s rate -v -L %(SAMPLERATE)s dither' % transcode_args]
else:
commands = map(lambda cmd: cmd % transcode_args, transcoding_steps)
return commands
def run(cmd, check=True, env=None, inputtext=None, logfiles=None):
"""
Run a command, dumping it cut-n-pasteably if required. Checks the return
code unless check=False. Returns a dictionary of stdout, stderr and return
code (rc)
"""
if logfiles is None:
logfiles = []
logging.debug("running command: %s",
(" ".join([pipes.quote(word) for word in cmd])))
if env is None:
env = os.environ.copy()
proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
[stdout, stderr] = proc.communicate(inputtext)
if check and proc.returncode != 0:
logging.error("command failed: %s",
(" ".join([pipes.quote(word) for word in cmd])))
logging.error("stdout: %s", stdout)
logging.error("stderr: %s", stderr)
for log_path in logfiles:
with open(log_path) as log_file:
logging.error("%s:\n%s", log_path, log_file.read())
raise Exception
return {"stdout": stdout, "stderr": stderr, "rc": proc.returncode}
def copy(source, destination, recursive=False, use_sudo=False):
"""
Copy a file or directory
"""
func = use_sudo and run_as_root or run
options = '-r ' if recursive else ''
func('/bin/cp {0}{1} {2}'.format(options, quote(source), quote(destination)))
def put_file(self, in_path, out_path):
""" transfer a file from local to remote """
super(Connection, self).put_file(in_path, out_path)
display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self.host)
if not os.path.exists(to_bytes(in_path, errors="strict")):
raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path)))
# scp and sftp require square brackets for IPv6 addresses, but
# accept them for hostnames and IPv4 addresses too.
host = "[%s]" % self.host
if C.DEFAULT_SCP_IF_SSH:
cmd = self._build_command("scp", in_path, u"{0}:{1}".format(host, pipes.quote(out_path)))
in_data = None
else:
cmd = self._build_command("sftp", to_bytes(host))
in_data = u"put {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path))
in_data = to_bytes(in_data, nonstring="passthru")
(returncode, stdout, stderr) = self._run(cmd, in_data)
if returncode != 0:
raise AnsibleError(
"failed to transfer file to {0}:\n{1}\n{2}".format(to_str(out_path), to_str(stdout), to_str(stderr))
)
def start_interaction_thread(push, banner, prompt):
""" Run the python script termapp_client.py (which must be in the current
directory) in a Terminal window. Set its current directory to the
current directory, and pass as an argument to it the name of a temporary
directory where it should look for named pipes to communcate with.
Arguments:
push -- a function which takes a line of input and returns True if the
secondary prompt should be used next and False if the primary prompt
should be used. push should use sys.stdout and sys.stderr for its
output, but reading from sys.stdin will just give it EOF.
banner -- a message to print before the first prompt
prompt -- a prefix for the prompt, " >> " will be added to make the
primary prompt and "... " for the secondary
"""
path = tempfile.mkdtemp()
app = appscript.app("Terminal")
app.do_script("cd {0};python termapp_client.py {1};exit".format(
quote(os.getcwd()), quote(path)))
app.activate()
t = Thread(target=run_server, name="console",
args=(push, banner, prompt, path))
t.setDaemon(True)
t.start()
return t
def recordVideo(self):
jobpath = self.config.dirname + "/" + self.jobname
if os.path.exists(jobpath):
shutil.rmtree(jobpath)
os.mkdir( jobpath , 0755 )
for cam in self.selectedCams:
# Get Address
address = self.config.cameras.cameras[cam].address
vidpath = jobpath + "/" + cam
if os.path.exists(vidpath ):
pass
else:
os.mkdir( vidpath , 0755 )
vidpath = pipes.quote(vidpath)
vidpath ="{}".format(vidpath)
livepath = os.path.dirname(os.path.realpath(__file__)) + "/../packages/live/testProgs/openRTSP"
livepath = pipes.quote(livepath)
# Create .cal file out of calibration data.
cmd = "{} -v -i -L ".format(livepath) +vidpath + " "+ address
print cmd
proc = subprocess.Popen(cmd, universal_newlines=True, shell=True, executable="/bin/bash", stdin=subprocess.PIPE)
self.procList.append(proc)
exit_codes = [p.wait() for p in self.procList]
def remotePath(abspath, host = 'tin', root = 'comp'):
'''Get the location of the a file on the remote file system
from one of the roots ['compbio', 'programming']'''
if root == 'prog':
subfun = progPath
subterm = 'progPath'
else:
subfun = compPath
subterm = 'compPath'
if host == None:
#IS THIS WHY THE LOCAL CALLS TO BSUB ARE FAILING?
return subfun(abspath).strip()
scr = pipes.quote('''
echo `python -c {0}`'''.format(pipes.quote('''
import compbio.config as config
import os, inspect
print config.{1}('{0}', absolute = True)
'''.format( subfun(abspath),
subterm
))))
ssh_scr = 'ssh {1} {0}'.format(scr, host)
out = spc.Popen(ssh_scr, shell = True, stdout = spc.PIPE).\
communicate()[0]
return out.strip()
def install_requirements(requirements, install_prefix=ENVIRONMENT):
"""
Manually install all requirements from binary distributions.
:param requirements: A list of :py:class:`pip_accel.req.Requirement` objects.
:param install_prefix: The "prefix" under which the requirements should be
installed. This will be a pathname like ``/usr``,
``/usr/local`` or the pathname of a virtual
environment.
:returns: ``True`` if it succeeds in installing all requirements from
binary distribution archives, ``False`` otherwise.
"""
install_timer = Timer()
logger.info("Installing from binary distributions ..")
python = os.path.join(install_prefix, 'bin', 'python')
pip = os.path.join(install_prefix, 'bin', 'pip')
for requirement in requirements:
if os.system('%s uninstall --yes %s >/dev/null 2>&1' % (pipes.quote(pip), pipes.quote(requirement.name))) == 0:
logger.info("Uninstalled previously installed package %s.", requirement.name)
members = get_binary_dist(requirement.name, requirement.version,
requirement.source_directory, requirement.url,
prefix=install_prefix, python=python)
install_binary_dist(members, prefix=install_prefix, python=python)
logger.info("Finished installing all requirements in %s.", install_timer)
return True
def _fix_type(self, status):
self.node.run("rm -rf -- {}".format(quote(self.name)))
self.node.run("mkdir -p -- {}".format(quote(self.name)))
if self.attributes['mode']:
self._fix_mode(status)
if self.attributes['owner'] or self.attributes['group']:
self._fix_owner(status)
def addBonding(self, bonding, bridge=None, bondingOptions=None, mtu=None,
ipaddr=None, netmask=None, gateway=None, bootproto=None,
onboot='yes', **kwargs):
""" Create ifcfg-* file with proper fields for bond """
if not bondingOptions:
bondingOptions = 'mode=802.3ad miimon=150'
conf = 'BONDING_OPTS=%s\n' % pipes.quote(bondingOptions or '')
if bridge:
conf += 'BRIDGE=%s\n' % pipes.quote(bridge)
if netinfo.NetInfo().ifaceUsers(bonding):
confParams = netinfo.getIfaceCfg(bonding)
if not ipaddr:
ipaddr = confParams.get('IPADDR', None)
netmask = confParams.get('NETMASK', None)
gateway = confParams.get('GATEWAY', None)
bootproto = confParams.get('BOOTPROTO', None)
if not mtu:
mtu = confParams.get('MTU', None)
if mtu:
mtu = int(mtu)
self._createConfFile(conf, bonding, ipaddr, netmask, gateway,
bootproto, mtu, onboot, **kwargs)
# create the bonding device to avoid initscripts noise
if bonding not in open(netinfo.BONDING_MASTERS).read().split():
open(netinfo.BONDING_MASTERS, 'w').write('+%s\n' % bonding)
def addNic(self, nic, bonding=None, bridge=None, mtu=None,
ipaddr=None, netmask=None, gateway=None, bootproto=None,
onboot='yes', **kwargs):
""" Create ifcfg-* file with proper fields for NIC """
_netinfo = netinfo.NetInfo()
hwaddr = (_netinfo.nics[nic].get('permhwaddr') or
_netinfo.nics[nic]['hwaddr'])
conf = 'HWADDR=%s\n' % pipes.quote(hwaddr)
if bridge:
conf += 'BRIDGE=%s\n' % pipes.quote(bridge)
if bonding:
conf += 'MASTER=%s\nSLAVE=yes\n' % pipes.quote(bonding)
if _netinfo.ifaceUsers(nic):
confParams = netinfo.getIfaceCfg(nic)
if not ipaddr:
ipaddr = confParams.get('IPADDR', None)
netmask = confParams.get('NETMASK', None)
gateway = confParams.get('GATEWAY', None)
bootproto = confParams.get('BOOTPROTO', None)
if not mtu:
mtu = confParams.get('MTU', None)
if mtu:
mtu = int(mtu)
self._createConfFile(conf, nic, ipaddr, netmask, gateway,
bootproto, mtu, onboot, **kwargs)
def _generate_shell_command(parsed_args):
shell_cmd = []
if parsed_args.mode == 'atftest' and not parsed_args.run_test_as_app:
target = '$instrument'
if parsed_args.start_test_component:
target = parsed_args.start_test_component
shell_cmd.extend(['am', 'instrument'])
# Set target test classes and packages.
# Note that, the name may contain '$' character, so we need to escape them
# here.
if parsed_args.run_test_classes:
shell_cmd.extend(
['-e', 'class', pipes.quote(parsed_args.run_test_classes)])
if parsed_args.run_test_packages:
shell_cmd.extend(
['-e', 'package', pipes.quote(parsed_args.run_test_packages)])
if parsed_args.atf_gtest_list:
shell_cmd.extend(
['-e', 'atf-gtest-list', pipes.quote(parsed_args.atf_gtest_list)])
if parsed_args.atf_gtest_filter:
shell_cmd.extend(
['-e', 'atf-gtest-filter',
pipes.quote(parsed_args.atf_gtest_filter)])
shell_cmd.extend(['-r', '-w', target, ';'])
shell_cmd.extend(['stop', ';'])
return shell_cmd
def get_pack_base_path(pack_name):
"""
Return full absolute base path to the content pack directory.
Note: This function looks for a pack in all the load paths and return path to the first pack
which matched the provided name.
If a pack is not found, we return a pack which points to the first packs directory (this is
here for backward compatibility reasons).
:param pack_name: Content pack name.
:type pack_name: ``str``
:rtype: ``str``
"""
if not pack_name:
return None
packs_base_paths = get_packs_base_paths()
for packs_base_path in packs_base_paths:
pack_base_path = os.path.join(packs_base_path, pipes.quote(pack_name))
pack_base_path = os.path.abspath(pack_base_path)
if os.path.isdir(pack_base_path):
return pack_base_path
# Path with the provided name not found
pack_base_path = os.path.join(packs_base_paths[0], pipes.quote(pack_name))
pack_base_path = os.path.abspath(pack_base_path)
return pack_base_path
def run(self):
adjcamposepath= os.path.dirname(os.path.realpath(__file__)) + "/../bin/adjustCamPose"
adjcamposepath = pipes.quote(adjcamposepath)
# rossourcepath = os.path.dirname(os.path.realpath(__file__)) +"/../ros_catkin_ws/install_isolated/setup.bash"
# rossourcepath = pipes.quote(rossourcepath)
arconfigpath = os.path.dirname(os.path.realpath(__file__)) +"/../packages/ARToolKitPlus-2.2.1/sample/data/markerboard_480-499.cfg"
arconfigpath = pipes.quote(arconfigpath)
markerID_str = ''
self.procList =[]
#store camera calibration for removal
calibrationFiles = []
jobpath = self.config.dirname + "/" + self.jobname
camjobpath = jobpath + "/" + self.selectedCam
print "camjobpath : %s" %camjobpath
if os.path.exists(camjobpath ):
pass
else:
os.makedirs( camjobpath , 0755 )
calpath = camjobpath +"/ar_calibration.cal"
camjobpath = pipes.quote(camjobpath)
self.config.cameras.cameras[self.selectedCam].calibration.writeToARFile(calpath)
calibrationFiles.append(calpath)
originFlag = 0
resultFlag = 0
# pass in calibration data:
address =self.config.cameras.cameras[self.selectedCam].address
# cmd = "source " +rossourcepath
cmd = adjcamposepath+ " -l "+camjobpath + " -s " + str(self.config.markersize) + " -a " + arconfigpath + " -i "+ address
print "Command : %s" %cmd
self.proc = subprocess.Popen(cmd, shell=True,preexec_fn=os.setsid)
self.pipein = open("/tmp/adjustcampose",'r')
exit_codes = self.proc.wait()
shutil.rmtree(jobpath)
def reset_exports(self):
"""Delete all export files."""
self.execute('sh', '-c',
'rm -f %s/*.conf' % pipes.quote(self.ganesha_export_dir))
self._mkindex()
def describe_quoted(self, value):
if value is None: return '...'
return ' '.join(
[pipes.quote(item) for item in self.describe_each(value)])
def describe_quoted(self, value):
return pipes.quote(self.describe(value))
def device_escape_path(self, path):
return pipes.quote(path)
def raw_command(cmd, capture=False, env=None, data=None, cwd=None, explain=False, stdin=None, stdout=None,
cmd_verbosity=1, str_errors='strict'):
"""
:type cmd: collections.Iterable[str]
:type capture: bool
:type env: dict[str, str] | None
:type data: str | None
:type cwd: str | None
:type explain: bool
:type stdin: file | None
:type stdout: file | None
:type cmd_verbosity: int
:type str_errors: str
:rtype: str | None, str | None
"""
if not cwd:
cwd = os.getcwd()
if not env:
env = common_environment()
cmd = list(cmd)
escaped_cmd = ' '.join(pipes.quote(c) for c in cmd)
display.info('Run command: %s' % escaped_cmd, verbosity=cmd_verbosity)
display.info('Working directory: %s' % cwd, verbosity=2)
program = find_executable(cmd[0], cwd=cwd, path=env['PATH'], required='warning')
if program:
display.info('Program found: %s' % program, verbosity=2)
for key in sorted(env.keys()):
display.info('%s=%s' % (key, env[key]), verbosity=2)
if explain:
return None, None
communicate = False
if stdin is not None:
data = None
communicate = True
elif data is not None:
stdin = subprocess.PIPE
communicate = True
if stdout:
communicate = True
if capture:
stdout = stdout or subprocess.PIPE
stderr = subprocess.PIPE
communicate = True
else:
stderr = None
start = time.time()
try:
process = subprocess.Popen(cmd, env=env, stdin=stdin, stdout=stdout, stderr=stderr, cwd=cwd)
except OSError as ex:
if ex.errno == errno.ENOENT:
raise ApplicationError('Required program "%s" not found.' % cmd[0])
raise
if communicate:
encoding = 'utf-8'
data_bytes = data.encode(encoding) if data else None
stdout_bytes, stderr_bytes = process.communicate(data_bytes)
stdout_text = stdout_bytes.decode(encoding, str_errors) if stdout_bytes else u''
stderr_text = stderr_bytes.decode(encoding, str_errors) if stderr_bytes else u''
else:
process.wait()
stdout_text, stderr_text = None, None
status = process.returncode
runtime = time.time() - start
display.info('Command exited with status %s after %s seconds.' % (status, runtime), verbosity=4)
if status == 0:
return stdout_text, stderr_text
raise SubprocessError(cmd, status, stdout_text, stderr_text, runtime)
def run_impl(self, opts, args, uuid, work_files):
# Log file, temporary
if hasattr(self.event_handler, "contexts"):
t_file = TemporaryFile()
log_context = ReporterContext(None, self.event_handler.VV, t_file)
self.event_handler.contexts[uuid] = log_context
# Check suite engine specific compatibility
self.suite_engine_proc.check_global_conf_compat()
# Suite name from the current working directory
if opts.conf_dir:
self.fs_util.chdir(opts.conf_dir)
opts.conf_dir = os.getcwd()
if opts.defines_suite:
suite_section = "jinja2:" + self.suite_engine_proc.SUITE_CONF
if not opts.defines:
opts.defines = []
for define in opts.defines_suite:
opts.defines.append("[" + suite_section + "]" + define)
# --remote=KEY=VALUE,...
if opts.remote:
# opts.name always set for remote.
return self._run_remote(opts, opts.name)
conf_tree = self.config_load(opts)
self.fs_util.chdir(conf_tree.conf_dirs[0])
suite_name = opts.name
if not opts.name:
suite_name = os.path.basename(os.getcwd())
# Automatic Rose constants
# ROSE_ORIG_HOST: originating host
# ROSE_VERSION: Rose version (not retained in run_mode=="reload")
# Suite engine version
jinja2_section = "jinja2:" + self.suite_engine_proc.SUITE_CONF
my_rose_version = ResourceLocator.default().get_version()
suite_engine_key = self.suite_engine_proc.get_version_env_name()
if opts.run_mode in ["reload", "restart"]:
prev_config_path = self.suite_engine_proc.get_suite_dir(
suite_name, "log", "rose-suite-run.conf")
prev_config = ConfigLoader()(prev_config_path)
suite_engine_version = prev_config.get_value(
["env", suite_engine_key])
else:
suite_engine_version = self.suite_engine_proc.get_version()
auto_items = {"ROSE_ORIG_HOST": self.host_selector.get_local_host(),
"ROSE_VERSION": ResourceLocator.default().get_version(),
suite_engine_key: suite_engine_version}
for key, val in auto_items.items():
requested_value = conf_tree.node.get_value(["env", key])
if requested_value:
if key == "ROSE_VERSION" and val != requested_value:
exc = VersionMismatchError(requested_value, val)
raise ConfigValueError(["env", key], requested_value, exc)
val = requested_value
else:
conf_tree.node.set(["env", key], val,
state=conf_tree.node.STATE_NORMAL)
conf_tree.node.set([jinja2_section, key], '"' + val + '"')
# See if suite is running or not
hosts = []
if opts.host:
hosts.append(opts.host)
if opts.run_mode == "reload":
suite_run_hosts = self.suite_engine_proc.get_suite_run_hosts(
None, suite_name)
if not suite_run_hosts:
raise SuiteNotRunningError(suite_name)
hosts = suite_run_hosts
else:
self.suite_engine_proc.check_suite_not_running(suite_name)
# Install the suite to its run location
suite_dir_rel = self._suite_dir_rel(suite_name)
# Unfortunately a large try/finally block to ensure a temporary folder
# created in validate only mode is cleaned up. Exceptions are not
# caught here
try:
if opts.validate_suite_only_mode:
temp_dir = mkdtemp()
suite_dir = os.path.join(temp_dir, suite_dir_rel)
os.makedirs(suite_dir, 0o0700)
else:
suite_dir = os.path.join(
os.path.expanduser("~"), suite_dir_rel)
suite_conf_dir = os.getcwd()
locs_conf = ConfigNode()
if opts.new_mode:
if os.getcwd() == suite_dir:
raise NewModeError("PWD", os.getcwd())
elif opts.run_mode in ["reload", "restart"]:
raise NewModeError("--run", opts.run_mode)
self.suite_run_cleaner.clean(suite_name)
if os.getcwd() != suite_dir:
if opts.run_mode == "run":
self._run_init_dir(opts, suite_name, conf_tree,
locs_conf=locs_conf)
os.chdir(suite_dir)
# Housekeep log files
now_str = None
if not opts.install_only_mode and not opts.local_install_only_mode:
now_str = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
self._run_init_dir_log(opts, now_str)
self.fs_util.makedirs("log/suite")
# Rose configuration and version logs
self.fs_util.makedirs("log/rose-conf")
run_mode = opts.run_mode
if run_mode not in ["reload", "restart", "run"]:
run_mode = "run"
mode = run_mode
if opts.validate_suite_only_mode:
mode = "validate-suite-only"
elif opts.install_only_mode:
mode = "install-only"
elif opts.local_install_only_mode:
mode = "local-install-only"
prefix = "rose-conf/%s-%s" % (strftime("%Y%m%dT%H%M%S"), mode)
# Dump the actual configuration as rose-suite-run.conf
ConfigDumper()(conf_tree.node, "log/" + prefix + ".conf")
# Install version information file
write_source_vc_info(
suite_conf_dir, "log/" + prefix + ".version", self.popen)
# If run through rose-stem, install version information files for
# each source tree if they're a working copy
if hasattr(opts, 'source') and hasattr(opts, 'project'):
for i, url in enumerate(opts.source):
if os.path.isdir(url):
write_source_vc_info(
url, "log/" + opts.project[i] + "-" + str(i) +
".version", self.popen)
for ext in [".conf", ".version"]:
self.fs_util.symlink(prefix + ext, "log/rose-suite-run" + ext)
# Move temporary log to permanent log
if hasattr(self.event_handler, "contexts"):
log_file_path = os.path.abspath(
os.path.join("log", "rose-suite-run.log"))
log_file = open(log_file_path, "ab")
temp_log_file = self.event_handler.contexts[uuid].handle
temp_log_file.seek(0)
log_file.write(temp_log_file.read())
self.event_handler.contexts[uuid].handle = log_file
temp_log_file.close()
# Process Environment Variables
environ = self.config_pm(conf_tree, "env")
# Process Files
cwd = os.getcwd()
for rel_path, conf_dir in conf_tree.files.items():
if (conf_dir == cwd or
any(fnmatchcase(os.sep + rel_path, exclude)
for exclude in self.SYNC_EXCLUDES) or
conf_tree.node.get(
["jinja2:" + rel_path]) is not None):
continue
# No sub-directories, very slow otherwise
if os.sep in rel_path:
rel_path = rel_path.split(os.sep, 1)[0]
target_key = self.config_pm.get_handler(
"file").PREFIX + rel_path
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get("source")
if source_node is None:
target_node.set(
["source"], os.path.join(
conf_dir, rel_path))
elif source_node.is_ignored():
continue
self.config_pm(conf_tree, "file",
no_overwrite_mode=opts.no_overwrite_mode)
# Process Jinja2 configuration
self.config_pm(conf_tree, "jinja2")
# Ask suite engine to parse suite configuration
# and determine if it is up to date (unchanged)
if opts.validate_suite_only_mode:
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_dir, opts.run_mode, opts.strict_mode,
debug_mode=True)
else:
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_name, opts.run_mode, opts.strict_mode,
opts.debug_mode)
finally:
# Ensure the temporary directory created is cleaned up regardless
# of success or failure
if opts.validate_suite_only_mode and os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
# Only validating so finish now
if opts.validate_suite_only_mode:
return
# Install share/work directories (local)
for name in ["share", "share/cycle", "work"]:
self._run_init_dir_work(
opts, suite_name, name, conf_tree, locs_conf=locs_conf)
if opts.local_install_only_mode:
return
# Install suite files to each remote [user@]host
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
uuid_file = os.path.abspath(name + uuid)
open(uuid_file, "w").close()
work_files.append(uuid_file)
# Install items to user@host
conf = ResourceLocator.default().get_conf()
auths = self.suite_engine_proc.get_tasks_auths(suite_name)
proc_queue = [] # [[proc, command, "ssh"|"rsync", auth], ...]
for auth in sorted(auths):
host = auth
if "@" in auth:
host = auth.split("@", 1)[1]
# Remote shell
command = self.popen.get_cmd("ssh", "-n", auth)
# Provide ROSE_VERSION and CYLC_VERSION in the environment
shcommand = "env ROSE_VERSION=%s %s=%s" % (
my_rose_version, suite_engine_key, suite_engine_version)
# Use login shell?
no_login_shell = self._run_conf(
"remote-no-login-shell", host=host, conf_tree=conf_tree)
if not no_login_shell or no_login_shell.lower() != "true":
shcommand += r""" bash -l -c '"$0" "$@"'"""
# Path to "rose" command, if applicable
rose_bin = self._run_conf(
"remote-rose-bin", host=host, conf_tree=conf_tree,
default="rose")
# Build remote "rose suite-run" command
shcommand += " %s suite-run -vv -n %s" % (rose_bin, suite_name)
for key in ["new", "debug", "install-only"]:
attr = key.replace("-", "_") + "_mode"
if getattr(opts, attr, None) is not None:
shcommand += " --%s" % key
if opts.log_keep:
shcommand += " --log-keep=%s" % opts.log_keep
if opts.log_name:
shcommand += " --log-name=%s" % opts.log_name
if not opts.log_archive_mode:
shcommand += " --no-log-archive"
shcommand += " --run=%s" % opts.run_mode
# Build --remote= option
shcommand += " --remote=uuid=%s" % uuid
if now_str is not None:
shcommand += ",now-str=%s" % now_str
host_confs = [
"root-dir",
"root-dir{share}",
"root-dir{share/cycle}",
"root-dir{work}"]
locs_conf.set([auth])
for key in host_confs:
value = self._run_conf(key, host=host, conf_tree=conf_tree)
if value is not None:
val = self.popen.list_to_shell_str([str(value)])
shcommand += ",%s=%s" % (key, pipes.quote(val))
locs_conf.set([auth, key], value)
command.append(shcommand)
proc = self.popen.run_bg(*command)
proc_queue.append([proc, command, "ssh", auth])
while proc_queue:
sleep(self.SLEEP_PIPE)
proc, command, command_name, auth = proc_queue.pop(0)
if proc.poll() is None: # put it back in proc_queue
proc_queue.append([proc, command, command_name, auth])
continue
ret_code = proc.wait()
out, err = proc.communicate()
if ret_code:
raise RosePopenError(command, ret_code, out, err)
if command_name == "rsync":
self.handle_event(out, level=Event.VV)
continue
else:
self.handle_event(out, level=Event.VV, prefix="[%s] " % auth)
for line in out.split("\n"):
if "/" + uuid == line.strip():
locs_conf.unset([auth])
break
else:
filters = {"excludes": [], "includes": []}
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
filters["excludes"].append(name + uuid)
target = auth + ":" + suite_dir_rel
cmd = self._get_cmd_rsync(target, **filters)
proc_queue.append(
[self.popen.run_bg(*cmd), cmd, "rsync", auth])
# Install ends
ConfigDumper()(locs_conf, os.path.join("log", "rose-suite-run.locs"))
if opts.install_only_mode:
return
elif opts.run_mode == "reload" and suite_conf_unchanged:
conf_name = self.suite_engine_proc.SUITE_CONF
self.handle_event(SkipReloadEvent(suite_name, conf_name))
return
# Start the suite
self.fs_util.chdir("log")
ret = 0
# FIXME: should sync files to suite host?
if opts.run_mode != "reload":
if opts.host:
hosts = [opts.host]
else:
names = shlex.split(
conf.get_value(["rose-suite-run", "hosts"], ""))
if names:
hosts += names
if (hosts and len(hosts) == 1 and
self.host_selector.is_local_host(hosts[0])):
host = "localhost"
elif hosts:
host = self.host_selector(hosts)[0][0]
else:
host = "localhost"
self.handle_event(SuiteHostSelectEvent(suite_name, run_mode, host))
# FIXME: values in environ were expanded in the localhost
self.suite_engine_proc.run(
suite_name, host, environ, opts.run_mode, args)
# Disconnect log file handle, so monitoring tool command will no longer
# be associated with the log file.
self.event_handler.contexts[uuid].handle.close()
self.event_handler.contexts.pop(uuid)
# Launch the monitoring tool
# Note: maybe use os.ttyname(sys.stdout.fileno())?
if os.getenv("DISPLAY") and host and opts.gcontrol_mode:
self.suite_engine_proc.gcontrol(suite_name, host)
return ret
def ShellQuote(value):
"""Escapes the string for the safe use inside shell command line."""
# TODO(user): replace pipes.quote with shlex.quote when time comes.
return pipes.quote(SmartUnicode(value))
def _get_command_string(command): """Returns a shell escaped command string.""" return ' '.join(pipes.quote(part) for part in command)
def test(
self,
redirect=False,
name="runtests",
commands=None,
ignore_outcome=None,
ignore_errors=None,
display_hash_seed=False,
):
if commands is None:
commands = self.envconfig.commands
if ignore_outcome is None:
ignore_outcome = self.envconfig.ignore_outcome
if ignore_errors is None:
ignore_errors = self.envconfig.ignore_errors
with self.session.newaction(self, name) as action:
cwd = self.envconfig.changedir
if display_hash_seed:
env = self._get_os_environ(is_test_command=True)
# Display PYTHONHASHSEED to assist with reproducibility.
action.setactivity(
name,
"PYTHONHASHSEED={!r}".format(env.get("PYTHONHASHSEED")))
for i, argv in enumerate(commands):
# have to make strings as _pcall changes argv[0] to a local()
# happens if the same environment is invoked twice
message = "commands[{}] | {}".format(
i, " ".join([pipes.quote(str(x)) for x in argv]))
action.setactivity(name, message)
# check to see if we need to ignore the return code
# if so, we need to alter the command line arguments
if argv[0].startswith("-"):
ignore_ret = True
if argv[0] == "-":
del argv[0]
else:
argv[0] = argv[0].lstrip("-")
else:
ignore_ret = False
try:
self._pcall(
argv,
cwd=cwd,
action=action,
redirect=redirect,
ignore_ret=ignore_ret,
is_test_command=True,
)
except tox.exception.InvocationError as err:
if ignore_outcome:
msg = "command failed but result from testenv is ignored\ncmd:"
self.session.report.warning("{} {}".format(msg, err))
self.status = "ignored failed command"
continue # keep processing commands
self.session.report.error(str(err))
self.status = "commands failed"
if not ignore_errors:
break # Don't process remaining commands
except KeyboardInterrupt:
self.status = "keyboardinterrupt"
self.session.report.error(self.status)
raise
def process(self, inputs, outputs):
# Benchmarking
start_time = system.now_raw()
log.ODM_INFO('Running ODM Georeferencing Cell')
# get inputs
args = inputs.args
tree = inputs.tree
reconstruction = inputs.reconstruction
gcpfile = tree.odm_georeferencing_gcp
doPointCloudGeo = True
transformPointCloud = True
verbose = '-verbose' if self.params.verbose else ''
geo_ref = reconstruction.georef
# check if we rerun cell or not
rerun_cell = (args.rerun is not None and
args.rerun == 'odm_georeferencing') or \
(args.rerun_all) or \
(args.rerun_from is not None and
'odm_georeferencing' in args.rerun_from)
runs = [{
'georeferencing_dir':
tree.odm_georeferencing,
'texturing_dir':
tree.odm_texturing,
'model':
os.path.join(tree.odm_texturing, tree.odm_textured_model_obj)
}]
if args.skip_3dmodel:
runs = []
if not args.use_3dmesh:
# Make sure 2.5D mesh is georeferenced before the 3D mesh
# Because it will be used to calculate a transform
# for the point cloud. If we use the 3D model transform,
# DEMs and orthophoto might not align!
runs.insert(
0, {
'georeferencing_dir':
tree.odm_25dgeoreferencing,
'texturing_dir':
tree.odm_25dtexturing,
'model':
os.path.join(tree.odm_25dtexturing,
tree.odm_textured_model_obj)
})
for r in runs:
odm_georeferencing_model_obj_geo = os.path.join(
r['texturing_dir'], tree.odm_georeferencing_model_obj_geo)
odm_georeferencing_log = os.path.join(r['georeferencing_dir'],
tree.odm_georeferencing_log)
odm_georeferencing_transform_file = os.path.join(
r['georeferencing_dir'],
tree.odm_georeferencing_transform_file)
odm_georeferencing_model_txt_geo_file = os.path.join(
r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo)
if not io.file_exists(odm_georeferencing_model_obj_geo) or \
not io.file_exists(tree.odm_georeferencing_model_laz) or rerun_cell:
# odm_georeference definitions
kwargs = {
'bin': context.odm_modules_path,
'bundle': tree.opensfm_bundle,
'imgs': tree.dataset_raw,
'imgs_list': tree.opensfm_bundle_list,
'model': r['model'],
'log': odm_georeferencing_log,
'input_trans_file': tree.opensfm_transformation,
'transform_file': odm_georeferencing_transform_file,
'coords': tree.odm_georeferencing_coords,
'output_pc_file': tree.odm_georeferencing_model_laz,
'geo_sys': odm_georeferencing_model_txt_geo_file,
'model_geo': odm_georeferencing_model_obj_geo,
'gcp': gcpfile,
'verbose': verbose
}
if args.fast_orthophoto:
kwargs['input_pc_file'] = os.path.join(
tree.opensfm, 'reconstruction.ply')
elif args.use_opensfm_dense:
kwargs['input_pc_file'] = tree.opensfm_model
else:
kwargs['input_pc_file'] = tree.smvs_model
if transformPointCloud:
kwargs[
'pc_params'] = '-inputPointCloudFile {input_pc_file} -outputPointCloudFile {output_pc_file}'.format(
**kwargs)
if geo_ref and geo_ref.projection and geo_ref.projection.srs:
kwargs[
'pc_params'] += ' -outputPointCloudSrs %s' % pipes.quote(
geo_ref.projection.srs)
else:
log.ODM_WARNING(
'NO SRS: The output point cloud will not have a SRS.'
)
else:
kwargs['pc_params'] = ''
# Check to see if the GCP file exists
if not self.params.use_exif and (self.params.gcp_file or
tree.odm_georeferencing_gcp):
log.ODM_INFO('Found %s' % gcpfile)
try:
system.run(
'{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys} -gcpFile {gcp} '
'-outputCoordFile {coords}'.format(**kwargs))
except Exception:
log.ODM_EXCEPTION('Georeferencing failed. ')
return ecto.QUIT
elif io.file_exists(
tree.opensfm_transformation) and io.file_exists(
tree.odm_georeferencing_coords):
log.ODM_INFO(
'Running georeferencing with OpenSfM transformation matrix'
)
system.run(
'{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'
.format(**kwargs))
elif io.file_exists(tree.odm_georeferencing_coords):
log.ODM_INFO(
'Running georeferencing with generated coords file.')
system.run(
'{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'
.format(**kwargs))
else:
log.ODM_WARNING(
'Georeferencing failed. Make sure your '
'photos have geotags in the EXIF or you have '
'provided a GCP file. ')
doPointCloudGeo = False # skip the rest of the georeferencing
if doPointCloudGeo:
# update images metadata
geo_ref.extract_offsets(
odm_georeferencing_model_txt_geo_file)
reconstruction.georef = geo_ref
# XYZ point cloud output
if args.pc_csv:
log.ODM_INFO(
"Creating geo-referenced CSV file (XYZ format)")
system.run(
"pdal translate -i \"{}\" "
"-o \"{}\" "
"--writers.text.format=csv "
"--writers.text.order=\"X,Y,Z\" "
"--writers.text.keep_unspecified=false ".format(
tree.odm_georeferencing_model_laz,
tree.odm_georeferencing_xyz_file))
# LAS point cloud output
if args.pc_las:
log.ODM_INFO("Creating geo-referenced LAS file")
system.run("pdal translate -i \"{}\" "
"-o \"{}\" ".format(
tree.odm_georeferencing_model_laz,
tree.odm_georeferencing_model_las))
if args.crop > 0:
log.ODM_INFO(
"Calculating cropping area and generating bounds shapefile from point cloud"
)
cropper = Cropper(tree.odm_georeferencing,
'odm_georeferenced_model')
decimation_step = 40 if args.fast_orthophoto or args.use_opensfm_dense else 90
# More aggressive decimation for large datasets
if not args.fast_orthophoto:
decimation_step *= int(
len(reconstruction.photos) / 1000) + 1
cropper.create_bounds_shapefile(
tree.odm_georeferencing_model_laz,
args.crop,
decimation_step=decimation_step,
outlier_radius=20 if args.fast_orthophoto else 2)
# Do not execute a second time, since
# We might be doing georeferencing for
# multiple models (3D, 2.5D, ...)
doPointCloudGeo = False
transformPointCloud = False
else:
log.ODM_WARNING('Found a valid georeferenced model in: %s' %
tree.odm_georeferencing_model_laz)
outputs.reconstruction = reconstruction
if args.time:
system.benchmark(start_time, tree.benchmarking, 'Georeferencing')
log.ODM_INFO('Running ODM Georeferencing Cell - Finished')
return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
def commandline(self):
line = '/usr/bin/lockf -s -t 0 -k "%s" /usr/local/bin/rsync' % (
self.rsync_path
)
if self.rsync_recursive:
line += ' -r'
if self.rsync_times:
line += ' -t'
if self.rsync_compress:
line += ' -z'
if self.rsync_archive:
line += ' -a'
if self.rsync_preserveperm:
line += ' -p'
if self.rsync_preserveattr:
line += ' -X'
if self.rsync_delete:
line += ' --delete-delay'
if self.rsync_delayupdates:
line += ' --delay-updates'
if self.rsync_extra:
line += ' %s' % self.rsync_extra
# Do not use username if one is specified in host field
# See #5096 for more details
if '@' in self.rsync_remotehost:
remote = self.rsync_remotehost
else:
remote = '"%s"@%s' % (
self.rsync_user,
self.rsync_remotehost,
)
if self.rsync_mode == 'module':
if self.rsync_direction == 'push':
line += ' "%s" %s::"%s"' % (
self.rsync_path,
remote,
self.rsync_remotemodule,
)
else:
line += ' %s::"%s" "%s"' % (
remote,
self.rsync_remotemodule,
self.rsync_path,
)
else:
line += (
' -e "ssh -p %d -o BatchMode=yes '
'-o StrictHostKeyChecking=yes"'
) % (
self.rsync_remoteport
)
if pipes.quote(self.rsync_remotepath) == self.rsync_remotepath:
rsync_remotepath = self.rsync_remotepath
else:
rsync_remotepath = '\\""%s"\\"' % self.rsync_remotepath
if self.rsync_direction == 'push':
line += ' "%s" %s:%s' % (
self.rsync_path,
remote,
rsync_remotepath,
)
else:
line += ' %s:%s "%s"' % (
remote,
rsync_remotepath,
self.rsync_path,
)
if self.rsync_quiet:
line += ' > /dev/null 2>&1'
return line
def value_quoted(kvs):
return {k: quote(v) for k, v in kvs.items()}
def _Quote(self, arg):
if arg.startswith('--') and '=' in arg:
prefix, value = arg.split('=', 1)
return pipes.quote(prefix) + '=' + pipes.quote(value)
return pipes.quote(arg)
def _prettify_cmd(cmd):
cmd = ' '.join(pipes.quote(x) for x in cmd)
if set(cmd) & set(['\x00', '\n']):
cmd = repr(cmd)
return cmd
def quote(command, *args):
if args:
return command % tuple(pipes.quote(a) for a in args)
else:
return command
def quote_args(seq):
return ' '.join(quote(arg) for arg in seq)
if notebook.match(full_name) and not ('.ipynb_checkpoints'
in full_name):
targets.append(os.path.abspath(full_name))
except FileNotFoundError:
pass
#
# convert to html
#
outdir = './html_notebooks'
if not os.path.exists(outdir):
os.makedirs(outdir)
try:
os.chdir(outdir)
for target in targets:
escaped = pipes.quote(target)
command = 'ipython nbconvert --to html {}'.format(escaped)
status, output = subprocess.getstatusoutput(command)
print(target, status, output)
except Exception as err:
print('trouble: {}'.format(err))
finally:
os.chdir('..')
#
# convert to latex
#
outdir = './pdf_notebooks'
if not os.path.exists(outdir):
os.makedirs(outdir)
try:
os.chdir(outdir)
def cmd_line(args):
"""build a command line that works in a shell.
"""
args = [str(x) for x in args]
return ' '.join(pipes.quote(x) for x in args)
def get_command_quoted(command):
"""Return shell quoted command string."""
return ' '.join(pipes.quote(part) for part in command)
print("Databases file not found...")
print("Starting backup of database " + DB_NAME)
multi = 0
# Starting actual database backup process.
if multi:
in_file = open(DB_NAME, "r")
flength = len(in_file.readlines())
in_file.close()
p = 1
dbfile = open(DB_NAME, "r")
while p <= flength:
db = dbfile.readline() # reading database name from file
db = db[:-1] # deletes extra line
dumpcmd = "mysqldump -h " + DB_HOST + " -u " + DB_USER + " -p" + DB_USER_PASSWORD + " " + db + " > " + pipes.quote(
TODAYBACKUPPATH) + "/" + db + ".sql"
os.system(dumpcmd)
compcmd = "tar -zcvf " + pipes.quote(
TODAYBACKUPPATH) + "/" + db + ".tar.gz " + pipes.quote(
TODAYBACKUPPATH) + "/" + db + ".sql"
os.system(compcmd)
delcmd = "rm " + pipes.quote(TODAYBACKUPPATH) + "/" + db + ".sql"
os.system(delcmd)
p = p + 1
dbfile.close()
else:
db = DB_NAME
dumpcmd = "mysqldump -h " + DB_HOST + " -u " + DB_USER + " -p" + DB_USER_PASSWORD + " " + db + " > " + pipes.quote(
TODAYBACKUPPATH) + "/" + db + ".sql"
os.system(dumpcmd)
compcmd = "tar -zcvf " + pipes.quote(
command = None
args = sys.argv
if len(args) > 1:
if tryDecode(args[1]) == 'start':
command = 'start'
elif tryDecode(args[1]) == 'stop':
command = 'stop'
if command:
args = args[2:]
if os.name == 'nt':
args = subprocess.list2cmdline(args[1:])
else:
import pipes # pipes module isn't available on Windows
args = " ".join([pipes.quote(tryDecode(v)) for v in args[1:]])
# HBase configuration folder path (where hbase-site.xml reside) for
# HBase/Phoenix client side property override
hbase_config_path = phoenix_utils.hbase_conf_dir
# default paths ## TODO: add windows support
java_home = os.getenv('JAVA_HOME')
hbase_pid_dir = os.path.join(tempfile.gettempdir(), 'phoenix')
phoenix_log_dir = os.path.join(tempfile.gettempdir(), 'phoenix')
phoenix_file_basename = 'phoenix-%s-traceserver' % getpass.getuser()
phoenix_log_file = '%s.log' % phoenix_file_basename
phoenix_out_file = '%s.out' % phoenix_file_basename
phoenix_pid_file = '%s.pid' % phoenix_file_basename
opts = os.getenv('PHOENIX_TRACESERVER_OPTS', '')
try:
open(path, "w")
exit_code = SUCCESS
except IOError, excpt:
exit_code = FAIL
logging.debug("Failed to create %s (%s)" % (path, excpt))
else:
infile = "/dev/urandom"
if size <= MB:
bs = size
count = 1
else:
bs = MB
count = int(size/MB)
cmd = "dd if=%s of=%s count=%d bs=%s" % (infile,
pipes.quote(path), count, bs)
(exit_code, _stdout, stderr) = run_cmd(cmd)
if exit_code:
logging.error("Failed to create %s (%s)" % (path, stderr))
return exit_code
def parse_command_line(argv):
'''Parse the command line, returning a dictionary of values'''
desc = 'Save a description of a directory tree, or create a new directory '\
'tree from a description file.'
epi = 'Ex: %s -d target_dir -o |-i description_file [options]' % sys.argv[0]
parser = argparse.ArgumentParser(description=desc, epilog=epi)
parser.add_argument('--debug', default=False, action='store_true',
def build_package(self):
dist_name = self.buildozer.config.get('app', 'package.name')
dist_dir = self.get_dist_dir(dist_name)
config = self.buildozer.config
package = self._get_package()
version = self.buildozer.get_version()
# add extra libs/armeabi files in dist/default/libs/armeabi
# (same for armeabi-v7a, x86, mips)
for config_key, lib_dir in (
('android.add_libs_armeabi', 'armeabi'),
('android.add_libs_armeabi_v7a', 'armeabi-v7a'),
('android.add_libs_x86', 'x86'),
('android.add_libs_mips', 'mips')):
patterns = config.getlist('app', config_key, [])
if not patterns:
continue
self.buildozer.debug('Search and copy libs for {}'.format(lib_dir))
for fn in self.buildozer.file_matches(patterns):
self.buildozer.file_copy(
join(self.buildozer.root_dir, fn),
join(dist_dir, 'libs', lib_dir, basename(fn)))
# update the project.properties libraries references
self._update_libraries_references(dist_dir)
# add src files
self._add_java_src(dist_dir)
# generate the whitelist if needed
self._generate_whitelist(dist_dir)
# build the app
build_cmd = [
("--name", quote(config.get('app', 'title'))),
("--version", version),
("--package", package),
("--sdk", config.getdefault('app', 'android.api',
self.android_api)),
("--minsdk", config.getdefault('app', 'android.minapi',
self.android_minapi)),
]
is_private_storage = config.getbooldefault(
'app', 'android.private_storage', True)
if is_private_storage:
build_cmd += [("--private", self.buildozer.app_dir)]
else:
build_cmd += [("--dir", self.buildozer.app_dir)]
# add permissions
permissions = config.getlist('app', 'android.permissions', [])
for permission in permissions:
# force the latest component to be uppercase
permission = permission.split('.')
permission[-1] = permission[-1].upper()
permission = '.'.join(permission)
build_cmd += [("--permission", permission)]
# meta-data
meta_datas = config.getlistvalues('app', 'android.meta_data', [])
for meta in meta_datas:
key, value = meta.split('=', 1)
meta = '{}={}'.format(key.strip(), value.strip())
build_cmd += [("--meta-data", meta)]
# add extra Java jar files
add_jars = config.getlist('app', 'android.add_jars', [])
for pattern in add_jars:
pattern = join(self.buildozer.root_dir, pattern)
matches = glob(expanduser(pattern.strip()))
if matches:
for jar in matches:
build_cmd += [("--add-jar", jar)]
else:
raise SystemError('Failed to find jar file: {}'.format(
pattern))
# add presplash
presplash = config.getdefault('app', 'presplash.filename', '')
if presplash:
build_cmd += [("--presplash", join(self.buildozer.root_dir,
presplash))]
# add icon
icon = config.getdefault('app', 'icon.filename', '')
if icon:
build_cmd += [("--icon", join(self.buildozer.root_dir, icon))]
# OUYA Console support
ouya_category = config.getdefault('app', 'android.ouya.category',
'').upper()
if ouya_category:
if ouya_category not in ('GAME', 'APP'):
raise SystemError(
'Invalid android.ouya.category: "{}" must be one of GAME or APP'.format(
ouya_category))
# add icon
ouya_icon = config.getdefault('app', 'android.ouya.icon.filename',
'')
build_cmd += [("--ouya-category", ouya_category)]
build_cmd += [("--ouya-icon", join(self.buildozer.root_dir,
ouya_icon))]
# add orientation
orientation = config.getdefault('app', 'orientation', 'landscape')
if orientation == 'all':
orientation = 'sensor'
build_cmd += [("--orientation", orientation)]
# fullscreen ?
fullscreen = config.getbooldefault('app', 'fullscreen', True)
if not fullscreen:
build_cmd += [("--window", )]
# wakelock ?
wakelock = config.getbooldefault('app', 'android.wakelock', False)
if wakelock:
build_cmd += [("--wakelock", )]
# intent filters
intent_filters = config.getdefault(
'app', 'android.manifest.intent_filters', '')
if intent_filters:
build_cmd += [("--intent-filters", join(self.buildozer.root_dir,
intent_filters))]
# build only in debug right now.
if self.build_mode == 'debug':
build_cmd += [("debug", )]
mode = 'debug'
else:
build_cmd += [("release", )]
mode = 'release'
self.execute_build_package(build_cmd)
try:
self.buildozer.hook("android_pre_build_apk")
self.execute_build_package(build_cmd)
self.buildozer.hook("android_post_build_apk")
except:
# maybe the hook fail because the apk is not
pass
# XXX found how the apk name is really built from the title
if exists(join(dist_dir, "build.gradle")):
# on gradle build, the apk use the package name, and have no version
packagename = config.get('app', 'package.name')
apk = u'{packagename}-{mode}.apk'.format(
packagename=packagename, mode=mode)
apk_dir = join(dist_dir, "build", "outputs", "apk")
apk_dest = u'{packagename}-{version}-{mode}.apk'.format(
packagename=packagename, mode=mode, version=version)
else:
# on ant, the apk use the title, and have version
bl = u'\'" ,'
apptitle = config.get('app', 'title')
if hasattr(apptitle, 'decode'):
apptitle = apptitle.decode('utf-8')
apktitle = ''.join([x for x in apptitle if x not in bl])
apk = u'{title}-{version}-{mode}.apk'.format(
title=apktitle,
version=version,
mode=mode)
apk_dir = join(dist_dir, "bin")
apk_dest = apk
# copy to our place
copyfile(join(apk_dir, apk), join(self.buildozer.bin_dir, apk_dest))
self.buildozer.info('Android packaging done!')
self.buildozer.info(
u'APK {0} available in the bin directory'.format(apk_dest))
self.buildozer.state['android:latestapk'] = apk_dest
self.buildozer.state['android:latestmode'] = self.build_mode
cmd = cfg.actions_dir + os.sep + action
if not os.path.realpath(cmd).startswith(cfg.actions_dir):
raise ValueError("Action has to be in directory %s" % cfg.actions_dir)
# contract 3C: interpret shell escape sequences as literal file names.
# contract 3E: fail if the action doesn't exist or exists elsewhere.
if not os.access(cmd, os.F_OK):
raise ValueError("Action must exist in action directory.")
cmd = [cmd]
# contract: 3C, 3D: don't allow users to insert escape characters in options
if options:
if not hasattr(options, "__iter__"):
options = [options]
cmd += [pipes.quote(option) for option in options]
# contract 1: commands can run via sudo.
if run_as_root:
cmd = ["sudo", "-n"] + cmd
# contract 3C: don't interpret shell escape sequences.
# contract 5 (and 6-ish).
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False)
if not async:
output, error = proc.communicate()
if proc.returncode != 0:
def test_cp_testfile(self):
'''
test salt-cp
'''
minions = []
for line in self.run_salt('--out yaml "*" test.ping'):
if not line:
continue
data = yaml.load(line)
minions.extend(data.keys())
self.assertNotEqual(minions, [])
testfile = os.path.abspath(
os.path.join(
os.path.dirname(os.path.dirname(__file__)),
'files', 'file', 'base', 'testfile'
)
)
testfile_contents = salt.utils.fopen(testfile, 'r').read()
for idx, minion in enumerate(minions):
ret = self.run_salt(
'--out yaml {0} file.directory_exists {1}'.format(
pipes.quote(minion), integration.TMP
)
)
data = yaml.load('\n'.join(ret))
if data[minion] is False:
ret = self.run_salt(
'--out yaml {0} file.makedirs {1}'.format(
pipes.quote(minion),
integration.TMP
)
)
data = yaml.load('\n'.join(ret))
self.assertTrue(data[minion])
minion_testfile = os.path.join(
integration.TMP, 'cp_{0}_testfile'.format(idx)
)
ret = self.run_cp('{0} {1} {2}'.format(
pipes.quote(minion),
pipes.quote(testfile),
pipes.quote(minion_testfile)
))
data = yaml.load('\n'.join(ret))
for part in data.values():
self.assertTrue(part[minion_testfile])
ret = self.run_salt(
'--out yaml {0} file.file_exists {1}'.format(
pipes.quote(minion),
pipes.quote(minion_testfile)
)
)
data = yaml.load('\n'.join(ret))
self.assertTrue(data[minion])
ret = self.run_salt(
'--out yaml {0} file.contains {1} {2}'.format(
pipes.quote(minion),
pipes.quote(minion_testfile),
pipes.quote(testfile_contents)
)
)
data = yaml.load('\n'.join(ret))
self.assertTrue(data[minion])
ret = self.run_salt(
'--out yaml {0} file.remove {1}'.format(
pipes.quote(minion),
pipes.quote(minion_testfile)
)
)
data = yaml.load('\n'.join(ret))
self.assertTrue(data[minion])
def Shell(self, *command): command = ' '.join([pipes.quote(arg) for arg in command]) return self._Execute([command], shell=True, ignore_interrupt_signal=True)
def make_become_cmd(self, cmd, executable=None):
""" helper function to create privilege escalation commands """
prompt = None
success_key = None
self.prompt = None
if executable is None:
executable = C.DEFAULT_EXECUTABLE
if self.become:
becomecmd = None
randbits = ''.join(random.choice(string.ascii_lowercase) for x in range(32))
success_key = 'BECOME-SUCCESS-%s' % randbits
success_cmd = pipes.quote('echo %s; %s' % (success_key, cmd))
# set executable to use for the privilege escalation method, with various overrides
exe = self.become_exe or \
getattr(self, '%s_exe' % self.become_method, None) or \
C.DEFAULT_BECOME_EXE or \
getattr(C, 'DEFAULT_%s_EXE' % self.become_method.upper(), None) or \
self.become_method
# set flags to use for the privilege escalation method, with various overrides
flags = self.become_flags or \
getattr(self, '%s_flags' % self.become_method, None) or \
C.DEFAULT_BECOME_FLAGS or \
getattr(C, 'DEFAULT_%s_FLAGS' % self.become_method.upper(), None) or \
''
if self.become_method == 'sudo':
# If we have a password, we run sudo with a randomly-generated
# prompt set using -p. Otherwise we run it with default -n, which makes
# it fail if it would have prompted for a password.
# Cannot rely on -n as it can be removed from defaults, which should be
# done for older versions of sudo that do not support the option.
#
# Passing a quoted compound command to sudo (or sudo -s)
# directly doesn't work, so we shellquote it with pipes.quote()
# and pass the quoted string to the user's shell.
# force quick error if password is required but not supplied, should prevent sudo hangs.
if self.become_pass:
prompt = '[sudo via ansible, key=%s] password: '******'%s %s -p "%s" -u %s %s -c %s' % (exe, flags.replace('-n',''), prompt, self.become_user, executable, success_cmd)
else:
becomecmd = '%s %s -u %s %s -c %s' % (exe, flags, self.become_user, executable, success_cmd)
elif self.become_method == 'su':
# passing code ref to examine prompt as simple string comparisson isn't good enough with su
def detect_su_prompt(data):
SU_PROMPT_LOCALIZATIONS_RE = re.compile("|".join(['(\w+\'s )?' + x + ' ?: ?' for x in SU_PROMPT_LOCALIZATIONS]), flags=re.IGNORECASE)
return bool(SU_PROMPT_LOCALIZATIONS_RE.match(data))
prompt = detect_su_prompt
becomecmd = '%s %s %s -c %s' % (exe, flags, self.become_user, pipes.quote('%s -c %s' % (executable, success_cmd)))
elif self.become_method == 'pbrun':
prompt='assword:'
becomecmd = '%s -b %s -u %s %s' % (exe, flags, self.become_user, success_cmd)
elif self.become_method == 'pfexec':
# No user as it uses it's own exec_attr to figure it out
becomecmd = '%s %s "%s"' % (exe, flags, success_cmd)
elif self.become_method == 'runas':
raise AnsibleError("'runas' is not yet implemented")
#FIXME: figure out prompt
# this is not for use with winrm plugin but if they ever get ssh native on windoez
becomecmd = '%s %s /user:%s "%s"' % (exe, flags, self.become_user, success_cmd)
elif self.become_method == 'doas':
prompt = 'Password:'******'doas'
if not self.become_pass:
flags += ' -n '
if self.become_user:
flags += ' -u %s ' % self.become_user
#FIXME: make shell independant
becomecmd = '%s %s echo %s && %s %s env ANSIBLE=true %s' % (exe, flags, success_key, exe, flags, cmd)
else:
raise AnsibleError("Privilege escalation method not found: %s" % self.become_method)
if self.become_pass:
self.prompt = prompt
self.success_key = success_key
return becomecmd
return cmd
def _setup_wrapper_script_content(self, setup, mrjob_tar_gz_name=None):
"""Return a (Bourne) shell script that runs the setup commands and then
executes whatever is passed to it (this will be our mapper/reducer).
We obtain a file lock so that two copies of the setup commands
cannot run simultaneously on the same machine (this helps for running
:command:`make` on a shared source code archive, for example).
"""
out = StringIO()
def writeln(line=''):
out.write(line + '\n')
# we're always going to execute this script as an argument to
# sh, so there's no need to add a shebang (e.g. #!/bin/sh)
writeln('# store $PWD')
writeln('__mrjob_PWD=$PWD')
writeln()
writeln('# obtain exclusive file lock')
# Basically, we're going to tie file descriptor 9 to our lockfile,
# use a subprocess to obtain a lock (which we somehow inherit too),
# and then release the lock by closing the file descriptor.
# File descriptors 10 and higher are used internally by the shell,
# so 9 is as out-of-the-way as we can get.
writeln('exec 9>/tmp/wrapper.lock.%s' % self._job_name)
# would use flock(1), but it's not always available
writeln("%s -c 'import fcntl; fcntl.flock(9, fcntl.LOCK_EX)'" %
cmd_line(self._opts['python_bin']))
writeln()
writeln('# setup commands')
# group setup commands so we can redirect their input/output (see
# below). Don't use parens; this would invoke a subshell, which would
# keep us from exporting environment variables to the task.
writeln('{')
for cmd in setup:
# reconstruct the command line, substituting $__mrjob_PWD/<name>
# for path dicts
line = ' ' # indent, since these commands are in a group
for token in cmd:
if isinstance(token, dict):
# it's a path dictionary
line += '$__mrjob_PWD/'
line += pipes.quote(self._working_dir_mgr.name(**token))
else:
# it's raw script
line += token
writeln(line)
# redirect setup commands' input/output so they don't interfere
# with the task (see Issue #803).
writeln('} 0</dev/null 1>&2')
writeln()
writeln('# release exclusive file lock')
writeln('exec 9>&-')
writeln()
writeln('# run task from the original working directory')
writeln('cd $__mrjob_PWD')
writeln('"$@"')
return out.getvalue()
n.comment('It is generated by ' + os.path.basename(__file__) + '.')
n.newline()
n.variable('ninja_required_version', '1.3')
n.newline()
n.comment('The arguments passed to configure.py, for rerunning it.')
configure_args = sys.argv[1:]
if '--bootstrap' in configure_args:
configure_args.remove('--bootstrap')
n.variable('configure_args', ' '.join(configure_args))
env_keys = set(['CXX', 'AR', 'CFLAGS', 'LDFLAGS'])
configure_env = dict((k, os.environ[k]) for k in os.environ if k in env_keys)
if configure_env:
config_str = ' '.join(
[k + '=' + pipes.quote(configure_env[k]) for k in configure_env])
n.variable('configure_env', config_str + '$ ')
n.newline()
CXX = configure_env.get('CXX', 'g++')
objext = '.o'
if platform.is_msvc():
CXX = 'cl'
objext = '.obj'
def src(filename):
return os.path.join('$root', 'src', filename)
def built(filename):
lilypond_binary = os.path.join('/usr/bin', 'lilypond')
# If we are called with full path, try to use lilypond binary
# installed in the same path; this is needed in GUB binaries, where
# @bindir is always different from the installed binary path.
if 'bindir' in globals() and bindir:
lilypond_binary = os.path.join(bindir, 'lilypond')
# Only use installed binary when we are installed too.
if '/usr/bin' == ('@' + 'bindir@') or not os.path.exists(lilypond_binary):
lilypond_binary = 'lilypond'
# Need to shell-quote, issue 3468
import pipes
lilypond_binary = pipes.quote(lilypond_binary)
global_options = None
def find_linestarts(s):
nls = [0]
start = 0
end = len(s)
while 1:
i = s.find('\n', start)
if i < 0:
break
i = i + 1
nls.append(i)
