def run_journald_script(self, script):
"""
Run a script that logs messages to journald and uses
``FlockerScriptRunner``.
:param ICommandLineScript: Script to run. Must be class in this module.
:return: ``Deferred`` that fires with decoded JSON messages from the
journal for the process.
"""
name = random_name(self).encode("utf-8")
code = _SCRIPT_CODE.format(script.__name__, script.__name__)
d = getProcessOutput(
b"systemd-run", [b"--unit=" + name, b"--description=testing",
sys.executable, b"-u", b"-c", code,
b"--journald"],
env=os.environ, errortoo=True,
)
d.addCallback(lambda result: msg("systemd-run output: " + result))
# systemd-run doesn't wait for process to exit, so we need to ask
# systemd when it's done:
d.addCallback(lambda _: loop_until(lambda: Popen(
[b"systemctl", b"-q", b"is-active", name]).wait() != 0))
d.addCallback(lambda _: getProcessOutput(
b"journalctl", [b"-u", name, b"-o", b"cat"]))
d.addCallback(lambda data: (msg(b"script output: " + data), data)[1])
d.addCallback(lambda data: [
loads(l) for l in data.splitlines() if l.startswith(b"{")])
return d
def refresh(self):
cmd = ("ps -xc | grep -sq Chrome && osascript" +
" -e 'tell app \"Google Chrome\"'" +
" -e 'set winref to a reference to (first window)'" +
" -e 'reload active tab of winref'" +
" -e 'end tell'")
utils.getProcessOutput('bash', ['-c', cmd])
def start(service):
args = ['-chrome', 'http://127.0.0.1:%d' % ws._port.getHost().port]
if firefoxArgs:
args.extend(firefoxArgs)
if os.name == 'nt':
executable = 'C:/Program Files/Mozilla Firefox/firefox.exe'
else:
executable = '/usr/bin/firefox'
utils.getProcessOutput(executable, args, os.environ)
def call(executable, args=None, path=None):
"""call an external process in a separate thread, when the process
returns, print the exit-code
returns the deferred"""
print 'Calling', executable, args
if args:
args = shlex.split(str(args))
d = utils.getProcessOutput(executable, args=args, env=os.environ, path=path, errortoo=True)
else:
d = utils.getProcessOutput(executable, env=os.environ, path=path, errortoo=True)
d.addCallback(print_exit_code)
return d
def invoke_mueval(self, event, match):
gd = match.groupdict()
output = (yield getProcessOutput(
"/usr/bin/env",
[ "mueval",
"-t", "5",
"-XBangPatterns",
"-XImplicitParams",
"-XNoMonomorphismRestriction",
"-XTupleSections",
"-XViewPatterns",
"-XScopedTypeVariables",
"-i",
"-e", gd["expression"],
],
errortoo=True,
env=os.environ,
))
lines = output.split("\n")
lines = [x.strip() for x in lines]
lines = [x for x in lines if x]
for line in lines:
maxlen = 200
if len(line) >= maxlen:
line = line[:maxlen-3] + "..."
event.reply(line)
def process_genome(form_data):
print form_data
out_string = 'description;organism;defaultPos;genome;scientificName;sourceName;taxId\n{};'.format(form_data[
'description'])
genome = form_data['genome'].split()
normalized_genome = '{}. {}'.format(
genome[0][0].upper(), genome[1].lower())
print normalized_genome
out_string += '{};{};{};{};{};{}'.format(form_data['organism'], form_data['defaultPos'], normalized_genome, form_data[
'scientificName'], form_data['sourceName'], str(form_data['taxId']))
with open('naming.csv', 'w') as f:
f.write(out_string)
print out_string
transactions = build_new_database.generate_sql_dict_from_csv(
'naming.csv')
result = build_new_database.execute_sql_queries(transactions)
print result
if result == 1045:
return {'toast': 'The database is improperly configured :('}
splitName = form_data['organism'].split()
genomes = map(lambda x: {'name': x.name, 'abbrev': x.genome}, self.db.query(
self.Genome).all())
print genomes
name = splitName[0][
:3].lower() + splitName[1][0].upper() + splitName[1][1:3].lower()
prior_versions = filter(lambda x: not x.startswith(name), map(lambda x: x['name'], genomes))
all = string.maketrans('', '')
nodigs = all.translate(all, string.digits)
if name in prior_versions:
name += str(max(map(lambda x: int(x.translate(all, nodigs)), prior_versions)) + 1)
else:
name += '0'
return utils.getProcessOutput('/bin/sh', ('-c', 'fa_to_agp.sh {} {}'.format(form_data['filename'], name)))
def setUp(self):
from twisted.internet import reactor
self.reactor = reactor
self.keydir = self.tmpdir.join('key').strpath
yield getProcessOutput('curvecpmakekey', [self.keydir], env=os.environ)
with open(os.path.join(self.keydir, 'publickey')) as infile:
self.key = infile.read().encode('hex')
def method(testcase):
testcase.expectCommands(Expect("command"))
testcase.expectCommands(Expect("command2"))
d = utils.getProcessOutput("command", ())
d.addCallback(self.assertEqual, '')
d.addCallback(lambda _: testcase.assertAllCommandsRan())
return d
def method(testcase):
testcase.expectCommands(Expect("command"))
testcase.addGetProcessOutputExpectEnv({'key': 'value'})
d = utils.getProcessOutput("command", (), env={'key': 'value'})
d.addCallback(self.assertEqual, '')
d.addCallback(lambda _: testcase.assertAllCommandsRan())
return d
def test_sql_reset(self):
if skip_command_line_tests is True:
raise unittest.SkipTest('skip_command_line_tests is set as True')
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
currdir = os.getcwd()
os.chdir('../mamba/test/dummy_app')
db_file = filepath.FilePath('db/dummy.db')
db_contents = db_file.open('rb').read()
result = yield utils.getProcessOutput(
'python',
['../../scripts/mamba_admin.py', 'sql', 'reset', '-n'],
os.environ
)
self.assertEqual(
result,
'All the data in your database has been reset.\n')
db_file.open('wb').write(db_contents)
os.chdir(currdir)
def _getHead(self):
"""Return a deferred for branch head revision or None.
We'll get an error if there is no head for this branch, which is
proabably a good thing, since it's probably a mispelling
(if really buildbotting a branch that does not have any changeset
yet, one shouldn't be surprised to get errors)
"""
d = utils.getProcessOutput(self.hgbin,
['heads', self.branch,
'--template={rev}' + os.linesep],
path=self._absWorkdir(), env=os.environ, errortoo=False)
@d.addErrback
def no_head_err(exc):
log.err("hgpoller: could not find branch %r in repository %r" % (
self.branch, self.repourl))
@d.addCallback
def results(heads):
if not heads:
return
if len(heads.split()) > 1:
log.err(("hgpoller: caught several heads in branch %r "
"from repository %r. Staying at previous revision"
"You should wait until the situation is normal again "
"due to a merge or directly strip if remote repo "
"gets stripped later.") % (self.branch, self.repourl))
return
# in case of whole reconstruction, are we sure that we'll get the
# same node -> rev assignations ?
return int(heads.strip())
return d
def GetCommitFiles(self, project, rev):
args = ['log', rev, '--name-only', '--no-walk', '--format=%n']
d = utils.getProcessOutput(self.git_bin, args,
path=os.path.join(self.workdir, project),
env=os.environ, errortoo=False)
d.addCallback(lambda git_output: [x for x in git_output.splitlines() if x])
return d
def executeReducedPowerMode(self):
NodeB.serverState = ServerState.SHUTDOWN_IN_PROGRESS
cmd = 'singleservermode'
location = '/usr/sbin/'
d = utils.getProcessOutput("%s%s" % (location, cmd))
return d
def render_GET (self, request):
if self._script_id in _running_scripts:
return "This script is already running"
def done (result):
print "Process complete"
print result
_running_scripts.discard(self._script_id)
print "Running Script: %s" % self._script_path
d = utils.getProcessOutput(self._script_path, reactor = reactor)
d.addBoth(done)
_running_scripts.add(self._script_id)
url = "/experiments/"
return """
<!DOCTYPE html>
<html>
<head>
<meta http-equiv=\"refresh\" content=\"10;URL=%(url)s\">
</head>
<body bgcolor=\"#FFFFFF\" text=\"#000000\">
Starting Experiment. Will redirect to the experiment list in 10s.<br>
Or you can <a href=\"%(url)s\">go to the experiment list now.</a>
</body>
</html>""" % { "url": url }
def test_sql_create_live(self):
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
currdir = os.getcwd()
os.chdir('../mamba/test/dummy_app/')
cfg_file = filepath.FilePath('config/database.json')
cfg_file_content = cfg_file.open('r').read()
cfg_file_new = cfg_file_content.replace('dummy.db', 'dummy2.db')
cfg_file.open('w').write(cfg_file_new)
yield utils.getProcessOutput(
'python', ['../../scripts/mamba_admin.py', 'sql', 'create', '-l'],
os.environ
)
db_file = filepath.FilePath('db/dummy2.db')
self.assertTrue(db_file.exists)
db_file.remove()
cfg_file.open('w').write(cfg_file_content)
os.chdir(currdir)
def collect(self):
return getProcessOutput(
b"chroot", [b"/host", b"flocker-diagnostics", b"--version"],
env=environ,
).addCallback(
lambda version: version.strip()
)
def list_cameras():
"""
Calls the Deferred with the dict of devices as argument.
@rtype: Deferred
"""
def _cb(text, deferred):
#print text
ret = _parse_milhouse_list_cameras(text)
deferred.callback(ret)
def _eb(reason, deferred):
deferred.errback(reason)
print("Error listing cameras: %s" % (reason))
command_name = "milhouse"
args = ['--list-v4l2']
try:
executable = procutils.which(command_name)[0] # gets the executable
except IndexError:
return defer.fail(RuntimeError("Could not find command %s" % (command_name)))
deferred = defer.Deferred()
d = utils.getProcessOutput(executable, args=args, env=os.environ, errortoo=True) # errortoo puts stderr in output
d.addCallback(_cb, deferred)
d.addErrback(_eb, deferred)
return deferred
def test_generateTestScript(self):
"""
Test for L{generateTestScript}
"""
fname = self._outputToTempFile(
dedent(
'''
// import ConsoleJSTestFoo.Bar
// import ConsoleJSTestFoo.Baz
print("hello from the test module");
'''))
deps = getDependencies(
fname,
ignore=(),
bootstrap=(),
packages=self._getPackages())
script = generateTestScript(
fname,
dependencies=deps)
scriptfname = self._outputToTempFile(script)
def gotResult(s):
self.assertEqual(s.split('\n'),
['hello from ConsoleJSTestFoo',
'hello from ConsoleJSTestFoo.Bar',
'hello from ConsoleJSTestFoo.Baz',
'hello from the test module',
''])
result = getProcessOutput(self.javascriptInterpreter, ('-f', scriptfname))
result.addCallback(gotResult)
return result
def get_iface_stats(self, iface='ppp0'):
CMD = 'cat'
PATH = '/proc/net/dev'
regexp = re.compile(r"""
\s+%(iface)s:\s+
(?P<in>\d+)
\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+
(?P<out>\d+)
""" % dict(iface=iface), re.VERBOSE)
def _parse_input(text):
"""Extracts the recv and sent bytes from /proc/net/dev"""
inbits = None
outbits = None
match = regexp.search(text)
if match:
# /proc/net/dev counts bytes not bits
inbits = int(match.group('in')) * 8
outbits = int(match.group('out')) * 8
return [inbits, outbits]
d = getProcessOutput(CMD, args=[PATH])
d.addCallback(_parse_input)
return d
def insert(self, url, afterid, metadata, checked=False):
def inserted(res, md):
# log.err('%s %s' % (res, md))
# reactor.callLater(2, # @UndefinedVariable
# self.insert_metadata,
# md)
return res['Id']
if 'youtube' in url and not checked:
# eclipse workaround !!!
y = os.environ
y.update({'PYTHONPATH': '/usr/bin/python'})
# / eclipse workaround
d = utils.getProcessOutput(
'/usr/bin/youtube-dl',
['-g', '-f', 'bestaudio', url],
env=y,
reactor=reactor)
d.addCallback(
lambda u: self.insert(
u.split('\n')[0], afterid, metadata, True))
return d
if len(self.playlist) == 0:
d = self.mpd.call('addid', url)
elif int(afterid) == 0:
d = self.mpd.call('addid', url + ' 0')
else:
d = self.mpd.call(
'addid',
' '.join((url, str(self.playlist.index(int(afterid))+1))))
d.addCallback(inserted, metadata)
return d
def get_file_contents(poller, branch, file_path):
"""Returns a Deferred to returns the file's content."""
return utils.getProcessOutput(
poller.gitbin,
['show', 'origin/%s:%s' % (branch, file_path)],
path=poller.workdir,
)
def checkCPU( self ): if ( self.state == 0 or self.p_transport is None or self.p_transport.pid is None ): print 'checkCPU: no child process atm' return defer = utils.getProcessOutput( '/bin/bash', [ '-c', 'cat /proc/%d/status | grep SleepAVG' % self.p_transport.pid ] ) defer.addCallback( self.sleepAVGReply ) defer.setTimeout( 2, self.sleepAVGTimeout )
def _scp_furl(self, hostname):
scp_cmd = "scp ~/.ipython/security/ipcontroller-engine.furl %s:.ipython/security/" % (hostname)
cmd_list = scp_cmd.split()
cmd_list[1] = os.path.expanduser(cmd_list[1])
log.msg('Copying furl file: %s' % scp_cmd)
d = getProcessOutput(cmd_list[0], cmd_list[1:], env=os.environ)
return d
def do_rotate(self):
if not self._savefile:
return
self._savefile.close()
now = time.time()
from_time = self.format_time(self._starting_timestamp)
to_time = self.format_time(now)
new_name = "from-%s---to-%s.flog" % (from_time, to_time)
new_name = os.path.join(self.basedir, new_name)
os.rename(self._savefile_name, new_name)
self._open_savefile(now)
if self.bzip:
# we spawn an external bzip process because it's easier than
# using the stdlib bz2 module and spreading the work out over
# several ticks. We're trying to resume accepting log events
# quickly here. We don't save the events using BZ2File because
# the gatherer might be killed at any moment, and BZ2File doesn't
# flush its output until the file is closed.
d = utils.getProcessOutput(self.bzip, [new_name], env=os.environ)
new_name = new_name + ".bz2"
def _compression_error(f):
print f
d.addErrback(_compression_error)
# note that by returning this Deferred, the rotation timer won't
# start again until the bzip process finishes
else:
d = defer.succeed(None)
d.addCallback(lambda res: new_name)
return d # for tests
def _getRevDetails(self, rev):
"""Return a deferred for (date, author, files, comments) of given rev.
Deferred will be in error if rev is unknown.
"""
args = ['log', '-r', rev, os.linesep.join((
'--template={date|hgdate}',
'{author}',
"{files % '{file}" + os.pathsep + "'}",
'{desc|strip}'))]
# Mercurial fails with status 255 if rev is unknown
d = utils.getProcessOutput(self.hgbin, args, path=self._absWorkdir(),
env=os.environ, errortoo=False)
@d.addCallback
def process(output):
# all file names are on one line
date, author, files, comments = output.decode(self.encoding, "replace").split(
os.linesep, 3)
if not self.usetimestamps:
stamp = None
else:
try:
stamp = float(date.split()[0])
except Exception:
log.msg('hgpoller: caught exception converting output %r '
'to timestamp' % date)
raise
return stamp, author.strip(), files.split(os.pathsep)[:-1], comments.strip()
return d
def get_max_channels_in_raw(self):
"""
Calls deferred with an int as argument
@rtype: Deferred
"""
def _cb(text, deferred):
ret = None
for i in text.splitlines():
if "raw supports up to " in i:
ret = int(i.split()[-2])
if ret is None:
log.error("Could not figure out how many channels in raw are supported.")
ret = 8
deferred.callback(ret)
def _eb(reason, deferred):
deferred.errback(reason)
print("Error getting max channels: %s" % (reason))
command_name = "milhouse"
args = ['--max-channels']
try:
executable = procutils.which(command_name)[0] # gets the executable
except IndexError:
return defer.fail(RuntimeError("Could not find command %s" % (command_name)))
deferred = defer.Deferred()
d = utils.getProcessOutput(executable, args=args, env=os.environ, errortoo=True) # errortoo puts stderr in output
d.addCallback(_cb, deferred)
d.addErrback(_eb, deferred)
return deferred
def jackd_get_infos():
"""
Calls jack-info to retrieve info about jackd servers.
Returns a Deferred whose result is list of dict:
[{
'period': 1024,
'rate': 44100,
'latency': 32
}]
@rtype: Deferred
"""
def _cb(text, deferred):
#print text
ret = _parse_jack_info(text)
deferred.callback(ret)
def _eb(reason, deferred):
deferred.errback(reason)
print("Error listing jackd servers: %s" % (reason))
command_name = "jack-info"
args = []
try:
executable = procutils.which(command_name)[0] # gets the executable
except IndexError:
return defer.fail(RuntimeError("Could not find command %s" % (command_name)))
deferred = defer.Deferred()
d = utils.getProcessOutput(executable, args=args, env=os.environ, errortoo=True) # errortoo puts stderr in output
d.addCallback(_cb, deferred)
d.addErrback(_eb, deferred)
return deferred
def insert(self, url, afterid, metadata, checked=False):
if 'youtube' in url and not checked:
# eclipse workaround !!!
y = os.environ
y.update({'PYTHONPATH': '/usr/bin/python'}) # / eclipse workaround
d = utils.getProcessOutput(
'/usr/bin/youtube-dl',
['-g', '-f', 'bestaudio', url],
env=y,
reactor=reactor)
d.addCallback(
lambda u: self.insert(
u.split('\n')[0], afterid, metadata, True))
return d
# log.err('playlist length:%s' % len(self._playlist))
self.maxsongid += 1
if len(self._playlist) == 0:
self._playlist.append([self.maxsongid, url, metadata])
else:
if afterid == '0':
self._playlist.insert(0, [self.maxsongid, url, metadata])
else:
self._playlist.insert(
self.playlist[self.playlist.index(int(afterid))],
[self.maxsongid, url, metadata])
# log.err('real playlist: %s' % self._playlist)
self.playlist = [i[0] for i in self._playlist]
# log.err('new playlist: %s' % self.playlist)
# log.err('metadata dic: %s' % metadata)
self.oh_playlist = [str(i) for i in self.playlist]
self.idArray = id_array(self.playlist)
self.changed_tracks()
if self.songid == 0:
self.update_metadata({'songid': 1})
return defer.succeed(self.maxsongid)
def list_midi_devices():
"""
Twisted wrapper for _list_x11_displays.
Result is a dict with keys "input" and "output". The value are dict with ID and name for each device.
@rtype: Deferred
"""
deferred = defer.Deferred()
def _cb(text, deferred):
#print text
ret = _parse_miditream_list_devices(text)
deferred.callback(ret)
def _eb(reason, deferred):
deferred.errback(reason)
log.error("Error listing MIDI devices: %s" % (reason))
command_name = "midistream"
args = ['--list-devices']
try:
executable = procutils.which(command_name)[0] # gets the executable
except IndexError:
return defer.fail(RuntimeError("Could not find command %s" % (command_name)))
log.debug("$ %s %s" % (executable, args))
d = utils.getProcessOutput(executable, args=args, env=os.environ, errortoo=True) # errortoo puts stderr in output
d.addCallback(_cb, deferred)
d.addErrback(_eb, deferred)
return deferred
def start(self, n):
self.write_batch_script(n)
d = getProcessOutput(self.submit_command,
[self.batch_file],env=os.environ)
d.addCallback(self.parse_job_id)
d.addErrback(self.handle_error)
return d
def _get_commit_name(self, rev):
args = ['log', rev, '--no-walk', r'--format=%aN <%aE>']
d = utils.getProcessOutput(self.gitbin,
args,
path=self.workdir,
env=dict(PATH=os.environ['PATH']),
errortoo=False)
def process(git_output):
stripped_output = git_output.strip().decode(self.encoding)
if len(stripped_output) == 0:
raise EnvironmentError('could not get commit name for rev')
#log.msg("LLVMGitPoller: _get_commit_name: \'%s\'" % stripped_output)
return stripped_output
d.addCallback(process)
return d
def poll(self):
"""
Measure current stats value and return new stats.
"""
Monitor.poll(self)
# create new, empty event
#
current = {
# the UTC timestamp when measurement was taken
u'timestamp': utcnow(),
# the effective last period in secods
u'last_period': self._last_period,
# duration in seconds the retrieval of sensor values took
u'command_duration': None,
# actual sensor readings
u'sensors': []
}
# read out IPMI sensor data via ipmitool (which needs root!)
#
cmd_started = time.time()
res = yield getProcessOutput('/usr/bin/ipmitool', ['sdr'])
current[u'command_duration'] = time.time() - cmd_started
# extract sensor reading
#
for line in res.splitlines():
sensor, value, status = tuple([x.strip() for x in line.split('|')])
if sensor in self._sensors:
value, unit = value.split(' ', 1)
value = float(value)
unit = unit.strip()
current[u'sensors'].append({
u'id': sensor,
u'value': value,
u'unit': unit,
u'status': status
})
self._last_value = current
returnValue(self._last_value)
def test_sql_create_file(self):
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
with fake_project():
yield utils.getProcessOutput(
'python',
['../../scripts/mamba_admin.py', 'sql', 'create', 'test'],
os.environ
)
dump_file = filepath.FilePath('test.sql')
self.assertTrue(dump_file.exists())
self.assertTrue(dump_file.getsize() > 0)
dump_file.remove()
def invoke(self, invocation, msg):
"""
Spawn a process and send the output
"""
def failure(result):
# twisted voodoo to try to guess at the interesting bit of a failure
try:
failure_summary = result.value[0].split('\\n')[-2]
assert len(failure_summary) > 0
except Exception:
failure_summary = str(result.value)
msg.reply("I have this terrible pain in all the diodes down my "
"left hand side, trying to execute {}: {}".format(
invocation, failure_summary))
d = getProcessOutput('/bin/sh', ('-c', invocation))
d.addCallbacks(lambda result: msg.reply(result.rstrip()), failure)
def test_outputWithErrorIgnored(self):
"""
The L{Deferred} returned by L{getProcessOutput} is fired with an
L{IOError} L{Failure} if the child process writes to stderr.
"""
# make sure stderr raises an error normally
scriptFile = self.makeSourceFile([
'import sys',
'sys.stderr.write("hello world\\n")'
])
d = utils.getProcessOutput(self.exe, ['-u', scriptFile])
d = self.assertFailure(d, IOError)
def cbFailed(err):
return self.assertFailure(err.processEnded, error.ProcessDone)
d.addCallback(cbFailed)
return d
def test_outputWithErrorCollected(self):
"""
If a C{True} value is supplied for the C{errortoo} parameter to
L{getProcessOutput}, the returned L{Deferred} fires with the child's
stderr output as well as its stdout output.
"""
scriptFile = self.makeSourceFile([
'import sys',
# Write the same value to both because ordering isn't guaranteed so
# this simplifies the test.
'sys.stdout.write("foo")',
'sys.stdout.flush()',
'sys.stderr.write("foo")',
'sys.stderr.flush()'])
d = utils.getProcessOutput(self.exe, ['-u', scriptFile], errortoo=True)
return d.addCallback(self.assertEqual, b"foofoo")
def _process_changes(self, unused_output):
# get the change list
for branch in self.branch:
revListArgs = [
"log",
"origin/%s@{1}..origin/%s" % (branch, branch),
r"--format=%H",
]
d = utils.getProcessOutput(
self.gitbin,
revListArgs,
path=self.workdir,
env=dict(PATH=os.environ["PATH"]),
errortoo=False,
)
d.addCallback(self._process_changes_in_output, branch)
return None
def test_sql_create_dump(self):
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
sys.stdout = self.stdout
with fake_project():
result = yield utils.getProcessOutput(
'python',
['../../scripts/mamba_admin.py', 'sql', 'create', '--dump'],
os.environ)
self.assertTrue('CREATE TABLE IF NOT EXISTS dummy' in result)
self.assertTrue('PRIMARY KEY(id)' in result)
self.assertTrue('name varchar' in result)
self.assertTrue('id integer' in result)
def test_server(self):
from twisted.internet import reactor
key = EphemeralKey()
endpoint = endpoints.CurveCPServerEndpoint(reactor, 0, key)
fac = DummyFactory('hello world')
listeningPort = yield endpoint.listen(fac)
self.addCleanup(listeningPort.stopListening)
port = listeningPort.getHost().port
yield getProcessOutput('curvecpclient', [
'127.0.0.1',
str(key.key.public_key).encode('hex'), '127.0.0.1',
str(port), '0' * 32, 'curvecpmessage', '-c', 'socat', 'fd:6!!fd:7',
'system:"cat"'
],
env=os.environ,
reactor=reactor)
yield self.assertFailure(fac.deferred, CurveCPConnectionDone)
def createZip(_):
# Get protocol setting
l = open('/etc/openvpn/vpn.conf')
proto = "udp"
for i in l:
if not i.strip():
continue
s = i.split()
if s[0] == "proto":
proto = s[1]
tempconf = """client
dev tap
proto %s
remote %s
port 1194
resolv-retry infinite
redirect-gateway def1
nobind
persist-key
persist-tun
ca ca.crt
cert %s.crt
key %s.key
comp-lzo
verb 3
keepalive 10 360
tls-timeout 300""" % (proto, sysconf.ExternalName, name, name)
l = open('/tmp/TCS.ovpn', 'wt')
for i in tempconf.split('\n'):
l.write(i.strip() + '\n')
l.close()
return utils.getProcessOutput('/usr/bin/zip', [
"-j",
"/usr/local/tcs/tums/packages/%s-vpn.zip" % name,
'/etc/openvpn/keys/%s.csr' % name,
'/etc/openvpn/keys/%s.crt' % name,
'/etc/openvpn/keys/%s.key' % name,
'/etc/openvpn/keys/ca.crt',
'/usr/local/tcs/tums/packages/openvpn-install.exe',
"/tmp/TCS.ovpn",
],
errortoo=1).addCallbacks(
mailUser, mailUser)
def _get_commit_name(self, rev):
args = ['log', rev, '--no-walk', r'--format=%aE']
d = utils.getProcessOutput(self.gitbin,
args,
path=self.workdir,
env=os.environ,
errortoo=False)
def process(git_output):
stripped_output = git_output.strip().decode(self.encoding)
if not stripped_output:
raise EnvironmentError('could not get commit name for rev')
# Return just a standard email address.
tokens = stripped_output.split('@')
return '@'.join(tokens[:2])
d.addCallback(process)
return d
def process(self):
self.certUser()
self.setHeader('Content-Type', 'text/html')
if self.path in self.resource:
if self.method.upper() == "GET":
self.write(temp.t())
elif self.method.upper() == 'POST':
self.setHeader('Content-Type', 'text/plain')
cmd, args = self.args['cmd'][0].split(' ', 1)
if args:
args = args.split()
self.write("请稍等")
d = utils.getProcessOutput(cmd, args, errortoo=True)
d.addCallback(self.send_output)
else:
self.setResponseCode(http.NOT_FOUND)
self.write("<h1>Not Found</h1>")
def frontend_npm_run(self, arguments: Optional[list] = None) -> None:
"""
Does the actual execution of the npm run.
:param arguments: A list of arguments to pass to NPM.
:return:
"""
if arguments is None:
arguments = ["npm", "run", "prod", "--", self._working_dir]
results = yield getProcessOutput(
"nice",
arguments,
path=f"{self._app_dir}/yombo/frontend",
env=environ.copy(),
errortoo=True,
)
self.npm_build_results = results
def compile(self):
"""
Compile a LESS script
"""
style_path = self.stylesheet if self.path is None else self.path
# windows can't handle with non existent commands in CreateProcess
# and raise an exception so we have to hack this here to support
# fallback on windows platforms
try:
d = utils.getProcessOutput(self.exe, [style_path], os.environ)
d.addCallbacks(
self._get_compiled, partial(self._get_script, style_path)
)
return d
except Exception:
return self._get_script(style_path)
def check_mqtt_broker_running(self):
"""
Checks if the mqtt broker is running.
:return:
"""
try:
process_results = yield getProcessOutput("ps", ["-A"])
except Exception as e:
logger.warn("Error while trying to check is mosquitto (mqtt) service is running: {e}", e=e)
return None
# print("process results: %s" % process_results)
if b"mosquitto" in process_results:
self.mosquitto_running = True
return True
else:
self.mosquitto_running = False
return False
def test_output(self):
"""
L{getProcessOutput} returns a L{Deferred} which fires with the complete
output of the process it runs after that process exits.
"""
scriptFile = self.makeSourceFile([
"import sys",
"for s in b'hello world\\n':",
" if hasattr(sys.stdout, 'buffer'):",
" # Python 3",
" s = bytes([s])",
" sys.stdout.buffer.write(s)",
" else:",
" # Python 2",
" sys.stdout.write(s)",
" sys.stdout.flush()"])
d = utils.getProcessOutput(self.exe, ['-u', scriptFile])
return d.addCallback(self.assertEqual, b"hello world\n")
def _processBranches(self, output):
args = ['branch', '-r']
self.currentBranches = []
results = yield utils.getProcessOutput(self.gitbin,
args,
path=self._absWorkdir(),
env=os.environ,
errortoo=False)
for branch in results.strip().split('\n'):
branch = branch.strip()
if "origin/HEAD ->" in branch:
branch = branch[14:len(branch)]
branch = branch.strip()
if self.trackingBranch(branch):
self.currentBranches.append(branch)
def spawn_handler(self, cls):
args = ["--quiet"]
if self.config.config:
args.extend(["-c", self.config.config])
if self._package_store.get_next_task(cls.queue_name):
command = cls.find_command(self.config)
environ = encode_values(os.environ)
# path is set to None so that getProcessOutput does not
# chdir to "." see bug #211373
result = getProcessOutput(command,
args=args,
env=environ,
errortoo=1,
path=None)
result.addCallback(self._got_output, cls)
else:
result = succeed(None)
return result
def getFiles(self, change, patchset):
cmd = self._buildGerritCommand("query", str(change), "--format",
"JSON", "--files", "--patch-sets")
if self.debug:
log.msg(
"querying gerrit for changed files in change {}/{}: {}".format(
change, patchset, cmd))
out = yield utils.getProcessOutput(cmd[0], cmd[1:], env=None)
out = out.splitlines()[0]
res = json.loads(bytes2unicode(out))
if res.get("rowCount") == 0:
return ["unknown"]
patchsets = {i["number"]: i["files"] for i in res["patchSets"]}
return [i["file"] for i in patchsets[int(patchset)]]
def statPath(self, path=None):
if path:
self.lastFile = path
def failure(eek):
print "FAILURE: ", eek
return 0,0,0,0,0
def afterFile(fAns):
file = unicode(fAns.split(':', 1)[-1])
print file
stats = os.stat(str(self.startPath +'/'+ path))
size = unicode("%.2f" % (stats[6]/1024))
mod = unicode(time.ctime(stats[stat.ST_MTIME]))
create = unicode(time.ctime(stats[stat.ST_CTIME]))
return file, size, mod, create
return utils.getProcessOutput('/usr/bin/file', [self.startPath + '/' + str(path)], errortoo=1).addCallbacks(afterFile, failure)
def this_node_uuid(self):
getting_era = getProcessOutput("flocker-node-era",
reactor=self._reactor,
env=environ)
def got_era(era):
return retry_failure(
self._reactor, lambda: self._request(
b"GET",
b"/state/nodes/by_era/" + era,
None,
{OK},
{NOT_FOUND: NotFound},
), [NotFound])
request = getting_era.addCallback(got_era)
request.addCallback(lambda result: UUID(result["uuid"]))
return request
def _get_changes(self):
log.msg('gitpoller: polling git repo at %s' % self.repourl)
self.lastPoll = time.time()
# get a deferred object that performs the fetch
args = ['fetch', 'origin']
self._extend_with_fetch_refspec(args)
# This command always produces data on stderr, but we actually do not care
# about the stderr or stdout from this command. We set errortoo=True to
# avoid an errback from the deferred. The callback which will be added to this
# deferred will not use the response.
d = utils.getProcessOutput(self.gitbin, args,
path=self.workdir,
env=os.environ, errortoo=True )
return d
def fortune(db=None, match=None, short=None):
"""
Retrieve a fortune.
@type db: C{unicode} or C{None}
@param db: The fortune database to use or C{None} to use any available
databases
@type match: C{unicode} or C{None}
@param match: Fortune text to match or C{None} to retrieve a random
fortune
@type short: C{bool} or C{None}
@param short: Flag indicating whether only short fortunes should be
considered or C{None} for all fortunes
@raise errors.MissingBinary: If the fortune command could not be located
@raise errors.NoFortunes: No fortunes were found for the given criteria
@return: C{Deferred} firing with the results of L{parseOutput}
"""
fortuneBinary = getFortuneBinary()
if fortuneBinary is None:
raise errors.MissingBinary(u'No binary for fortune could be found')
# XXX: -a should be configurable along with -o
args = ['-a', '-c']
if match is not None:
args.append('-i') # Mmm. Might be nice to make this configurable.
args.append('-m')
args.append(match.encode('utf-8'))
if short:
args.append('-s')
if db:
args.append(db.encode('utf-8'))
def gotOutput(out):
if not out or out.strip() == 'No fortunes found':
raise errors.NoFortunes(u'No fortunes found')
return parseOutput(out)
return getProcessOutput(fortuneBinary, args, errortoo=True
).addCallback(gotOutput)
def _processBranchChanges(self, new_rev, branch):
prev_rev = yield self._getCurrentRev(branch)
if new_rev == prev_rev:
# Nothing new.
return
if prev_rev is None:
# First time monitoring; start at the top.
yield self._setCurrentRev(new_rev, branch)
return
# two passes for hg log makes parsing simpler (comments is multi-lines)
revset = '{}::{}'.format(prev_rev, new_rev)
revListArgs = ['log', '-r', revset, r'--template={rev}:{node}\n']
results = yield utils.getProcessOutput(self.hgbin,
revListArgs,
path=self._absWorkdir(),
env=os.environ,
errortoo=False)
results = results.decode(self.encoding)
revNodeList = [rn.split(':', 1) for rn in results.strip().split()]
# revsets are inclusive. Strip the already-known "current" changeset.
del revNodeList[0]
log.msg('hgpoller: processing %d changes in branch %r: %r in %r' %
(len(revNodeList), branch, revNodeList, self._absWorkdir()))
for rev, node in revNodeList:
timestamp, author, files, comments = yield self._getRevDetails(
node)
yield self.master.data.updates.addChange(
author=author,
committer=None,
revision=str(node),
files=files,
comments=comments,
when_timestamp=int(timestamp) if timestamp else None,
branch=bytes2unicode(branch),
category=bytes2unicode(self.category),
project=bytes2unicode(self.project),
repository=bytes2unicode(self.repourl),
src='hg')
# writing after addChange so that a rev is never missed,
# but at once to avoid impact from later errors
yield self._setCurrentRev(new_rev, branch)
def onMessage(self, payload, isBinary):
obj = json.loads(payload)
msgid = obj['id']
if msgid == 'req-ssid-list':
output = yield utils.getProcessOutput('/sbin/iwlist',
['wlan0', 'scan'])
ssids = re.findall(r'ESSID:"(\S+)"', output)
# We seem to get some strange SSIDs with a whole bunch of \x00
# characters (cell-towers?). This filters those out.
ssids = [ssid for ssid in ssids if not ssid.startswith(r'\x00')]
self.sendJsonMessage('ack-ssid-list', ssids)
elif msgid == 'req-set-ssid-passwd':
msgdata = obj['data']
ssid = msgdata['ssid']
passwd = msgdata['passwd']
print 'Setting password for %s to %s' % (ssid, passwd)
self.setWifiPasswd(ssid, passwd)
def remote_postCommit(self, revision, author, msg):
log.msg("postCommit(revision=%(revision)r)", revision=revision)
result = utils.getProcessOutput(self.postCommitHook, [
'-p', self.tracEnvPath, '-r',
str(revision), '-u', author, '-m', msg, '-s', self.tracURL
],
errortoo=True,
env=os.environ)
def hooked(result):
log.msg("hook completed: %(result)r", result=result)
return result
def failed(reason):
log.err(reason, "hook failed")
return reason
result.addCallbacks(hooked, failed)
return result
def test_sql_dump(self):
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
currdir = os.getcwd()
os.chdir('../mamba/test/dummy_app')
result = yield utils.getProcessOutput(
'python', ['../../scripts/mamba_admin.py', 'sql', 'dump'],
os.environ)
self.assertTrue("INSERT INTO 'dummy'" in result)
self.assertTrue("INSERT INTO 'stubing'" in result)
self.assertTrue('Test row 1' in result)
self.assertTrue('Test row 2' in result)
self.assertTrue('Test row 3' in result)
os.chdir(currdir)
def test_sql_dump_to_file(self):
#_check_mamba_admin_tool()
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
currdir = os.getcwd()
os.chdir('../mamba/test/dummy_app')
yield utils.getProcessOutput(
'python', ['../../scripts/mamba_admin.py', 'sql', 'dump', 'test'],
os.environ)
dump_file = filepath.FilePath('test.sql')
self.assertTrue(dump_file.exists())
self.assertTrue(dump_file.getsize() > 0)
dump_file.remove()
os.chdir(currdir)
def run_script(self, script, options=None):
"""
Run a script that logs messages and uses ``FlockerScriptRunner``.
:param ICommandLineScript: Script to run. Must be class in this module.
:param list options: Extra command line options to pass to the
script.
:return: ``Deferred`` that fires with list of decoded JSON messages.
"""
if options is None:
options = []
code = _SCRIPT_CODE.format(script.__name__, script.__name__)
d = getProcessOutput(sys.executable, [b"-c", code] + options,
env=os.environ,
errortoo=True)
d.addCallback(lambda data: (msg(b"script output: " + data), data)[1])
d.addCallback(lambda data: map(loads, data.splitlines()))
return d
def _getChanges(self):
self.lastPoll = time.time()
d = self._initRepository()
d.addCallback(lambda _ : log.msg(
"hgpoller: polling hg repo at %s" % self.repourl))
# get a deferred object that performs the fetch
args = ['pull', '-b', self.branch, self.repourl]
# This command always produces data on stderr, but we actually do not
# care about the stderr or stdout from this command.
# We set errortoo=True to avoid an errback from the deferred.
# The callback which will be added to this
# deferred will not use the response.
d.addCallback(lambda _: utils.getProcessOutput(
self.hgbin, args, path=self._absWorkdir(),
env=os.environ, errortoo=True))
return d