def enable_ksm(enable, sudo):
if enable ^ check_ksm():
cmd = "echo {0:d} > /sys/kernel/mm/ksm/run".format(enable)
if sudo:
d = utils.getProcessValue(sudo,
["--", "su", "-c", cmd], env=os.environ)
else:
d = utils.getProcessValue(os.environ.get("SHELL", "/bin/sh"),
["-c", cmd], env=os.environ)
d.addCallback(_check_cb, cmd)
def cb_start_server_button_clicked(self, *args):
if hasattr(sys, "frozen"):
# TODO Find the absolute path.
def1 = utils.getProcessValue("slugathon.exe", ["server", "-n"],
env=os.environ)
else:
def1 = utils.getProcessValue(sys.executable,
["-m", "slugathon.net.Server", "-n"],
env=os.environ)
def1.addCallback(self.server_exited)
def1.addErrback(self.server_failed)
def cb_start_server_button_clicked(self, *args):
self.status_textview.modify_style(self.status_text_norm)
self.status_textview.get_buffer().set_text("Starting server")
if hasattr(sys, "frozen"):
# TODO Find the absolute path.
def1 = utils.getProcessValue("slugathon.exe", ["server", "-n"],
env=os.environ)
else:
def1 = utils.getProcessValue(sys.executable,
["-m", "slugathon.net.Server", "-n"],
env=os.environ)
# no errback called, irrespective of process return value
def1.addCallback(self.server_exited)
def _prepare_base_image(self):
"""
I am a private method for creating (possibly cheap) copies of a
base_image for start_instance to boot.
"""
if not self.base_image:
return
if self.cheap_copy:
clone_cmd = "qemu-img"
clone_args = "create -b %(base)s -f qcow2 %(image)s"
else:
clone_cmd = "cp"
clone_args = "%(base)s %(image)s"
clone_args = clone_args % {
"base": self.base_image,
"image": self.image,
}
log.msg("Cloning base image: {} {}'".format(clone_cmd, clone_args))
try:
rc = yield utils.getProcessValue(clone_cmd, clone_args.split())
log.msg("Cloning exit code was: {}".format(rc))
except Exception as e:
log.err("Cloning failed: {}".format(e))
raise
def jack_disconnect(source, sink):
"""
Calls jack_disconnect with the given arguments.
Returns a Deferred
"""
deferred = defer.Deferred()
def _cb(result):
if type(result) != int:
lor.error(
"The result of calling jack_disconnect should be an int.")
log.info("jack_disconnect result: " + str(result))
if result == 0:
deferred.callback(True)
else:
deferred.callback(False)
exec_name = "jack_disconnect"
try:
executable = procutils.which(exec_name)[0]
except IndexError:
log.error("Could not find executable %s" % (exec_name))
return defer.succeed(False)
else:
args = [source, sink]
log.info("$ %s %s" % (executable, " ".join(list(args))))
d = utils.getProcessValue(executable, args, os.environ, '.', reactor)
d.addCallback(_cb)
return deferred
def jack_disconnect(source, sink):
"""
Calls jack_disconnect with the given arguments.
Returns a Deferred
"""
deferred = defer.Deferred()
def _cb(result):
if type(result) != int:
lor.error("The result of calling jack_disconnect should be an int.")
log.info("jack_disconnect result: " + str(result))
if result == 0:
deferred.callback(True)
else:
deferred.callback(False)
exec_name = "jack_disconnect"
try:
executable = procutils.which(exec_name)[0]
except IndexError:
log.error("Could not find executable %s" % (exec_name))
return defer.succeed(False)
else:
args = [source, sink]
log.info("$ %s %s" % (executable, " ".join(list(args))))
d = utils.getProcessValue(executable, args, os.environ, '.', reactor)
d.addCallback(_cb)
return deferred
def _prepare_base_image(self):
"""
I am a private method for creating (possibly cheap) copies of a
base_image for start_instance to boot.
"""
if not self.base_image:
return defer.succeed(True)
if self.cheap_copy:
clone_cmd = "qemu-img"
clone_args = "create -b %(base)s -f qcow2 %(image)s"
else:
clone_cmd = "cp"
clone_args = "%(base)s %(image)s"
clone_args = clone_args % {
"base": self.base_image,
"image": self.image,
}
log.msg("Cloning base image: %s %s'" % (clone_cmd, clone_args))
d = utils.getProcessValue(clone_cmd, clone_args.split())
def _log_result(res):
log.msg("Cloning exit code was: %d" % res)
return res
def _log_error(err):
log.err("Cloning failed: %s" % err)
return err
d.addCallbacks(_log_result, _log_error)
return d
def run_once(executable, *args):
"""
Runs a command, without looking at its output or return value.
Returns a Deferred or None.
"""
from twisted.internet import reactor
global _original_environment_variables
def _cb(result):
#print(result)
pass
try:
executable = procutils.which(executable)[0]
except IndexError:
log.error("Could not find executable %s" % (executable))
return None
else:
env = _original_environment_variables
for k in ["GTK2_RC_FILES", "GTK_RC_FILES"]:
if env.has_key(k):
log.info("%s=%s" % (k, env[k]))
log.info("$ %s %s" % (executable, " ".join(list(args))))
log.debug("ENV=%s" % (env))
d = utils.getProcessValue(executable, args, env, '.', reactor)
d.addCallback(_cb)
return d
def test_sql_reset(self):
if skip_command_line_tests is True:
raise unittest.SkipTest('skip_command_line_tests is set as True')
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
currdir = os.getcwd()
os.chdir('../mamba/test/dummy_app')
db_file = filepath.FilePath('db/dummy.db')
db_contents = db_file.open('rb').read()
result = yield utils.getProcessOutput(
'python',
['../../scripts/mamba_admin.py', 'sql', 'reset', '-n'],
os.environ
)
self.assertEqual(
result,
'All the data in your database has been reset.\n')
db_file.open('wb').write(db_contents)
os.chdir(currdir)
def _doWork():
# Answer Set Programming file details
aspDir = "C:\Program Files\clingo-3.0.4-win32//"
ruleSetDir = "C:\Program Files\clingo-3.0.4-win32//"
aspFile = "ServerASPHelloWorld.lp"
# only one 'word' can be in the command.
# everything else gets split up and put in the args array
# if you were passing solver specific commands,
# you could also pass to args something like "-n 4"
# if you are in a unix-like environment, you will need to change 'clingo.exe' to just 'clingo'
command = aspDir + "clingo.exe"
args = [ruleSetDir + aspFile]
# run another command with getProcessOutput [to capture the stdout output]
# or with getProcessValue [to just capture the exit code of the command]
out = yield utils.getProcessOutput(command, args)
val = yield utils.getProcessValue(command, args)
# printing is the same as logging
print "getProcessOutput:", out
print "getProcessValue:", val
# here's how you get stuff to display back in the web browser
request.write("getProcessOutput: %s" % out)
# you can do as many request.writes as you like, whenever/whereever
request.write("<br />getProcessValue: %d" % val)
# finish the request so the browser doesn't expect more
request.finish()
# how you finish up a method that has the @defer.inlineCallbacks decorator
defer.returnValue(None)
def run(self, type, text):
d = getProcessValue(
"zenity",
args=(type, "--text", text),
env=os.environ
)
return d
def parallel(count=None):
coop = task.Cooperator()
work = (getProcessValue(executable) for i in range(10))
if count:
return defer.DeferredList([coop.coiterate(work) for i in xrange(count)])
else:
return coop.coiterate(work)
def test_sql_create_live(self):
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
currdir = os.getcwd()
os.chdir('../mamba/test/dummy_app/')
cfg_file = filepath.FilePath('config/database.json')
cfg_file_content = cfg_file.open('r').read()
cfg_file_new = cfg_file_content.replace('dummy.db', 'dummy2.db')
cfg_file.open('w').write(cfg_file_new)
yield utils.getProcessOutput(
'python', ['../../scripts/mamba_admin.py', 'sql', 'create', '-l'],
os.environ
)
db_file = filepath.FilePath('db/dummy2.db')
self.assertTrue(db_file.exists)
db_file.remove()
cfg_file.open('w').write(cfg_file_content)
os.chdir(currdir)
def stop(self, name):
""" Stop the container.
@param name: Name of the container which should be stopped.
@type name: str
@param command: Deferred whose callback is triggered on success
or whose errback is triggered on failure with
an error message.
@type command: twisted::Deferred
"""
log.msg("Stop container '{0}'".format(name))
deferred = Deferred()
try:
dfrd = getProcessValue('/usr/bin/lxc-stop', ('-n', name),
env=os.environ, reactor=self._reactor)
def cb(retVal):
if retVal == 0:
deferred.callback('Container successfully stopped.')
else:
e = ContainerError('Container could not be stopped: '
'Received exit code {0} from '
'lxc-stop.'.format(retVal))
deferred.errback(Failure(e))
dfrd.addCallback(cb)
except OSError:
e = ContainerError('Insufficient system resources to stop a '
'process.')
deferred.errback(Failure(e))
return deferred
def test_sql_reset(self):
if skip_command_line_tests is True:
raise unittest.SkipTest('skip_command_line_tests is set as True')
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
currdir = os.getcwd()
os.chdir('../mamba/test/dummy_app')
db_file = filepath.FilePath('db/dummy.db')
db_contents = db_file.open('rb').read()
result = yield utils.getProcessOutput(
'python',
['../../scripts/mamba_admin.py', 'sql', 'reset', '-n'],
os.environ
)
self.assertEqual(
result,
'All the data in your database has been reset.\n')
db_file.open('wb').write(db_contents)
os.chdir(currdir)
def test_is_mamba_package_for_egg_file(self):
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
with self._generate_docs():
self.config.parseOptions()
self.config['name'] = 'mamba-dummy'
self.packer.pack_application(
'bdist_egg', self.config,
config.Application('config/application.json')
)
major, minor = sys.version_info[:2]
self.assertTrue(os.path.exists(
'mamba_dummy-0.1.2-py{}.{}.egg'.format(major, minor))
)
path = filepath.FilePath(
'mamba_dummy-0.1.2-py{}.{}.egg'.format(major, minor)
)
is_mamba_package = self.packer.is_mamba_package(path)
self.assertTrue(is_mamba_package)
self.packer.do(
['rm', 'mamba_dummy-0.1.2-py{}.{}.egg'.format(major, minor)]
)
def test_sql_create_live(self):
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
currdir = os.getcwd()
os.chdir('../mamba/test/dummy_app/')
cfg_file = filepath.FilePath('config/database.json')
cfg_file_content = cfg_file.open('r').read()
cfg_file_new = cfg_file_content.replace('dummy.db', 'dummy2.db')
cfg_file.open('w').write(cfg_file_new)
yield utils.getProcessOutput(
'python', ['../../scripts/mamba_admin.py', 'sql', 'create', '-l'],
os.environ
)
db_file = filepath.FilePath('db/dummy2.db')
self.assertTrue(db_file.exists)
db_file.remove()
cfg_file.open('w').write(cfg_file_content)
os.chdir(currdir)
def _doWork():
# only one 'word' can be in the command.
# everything else gets split up and put in the args array
command = "pwd"
args = []
# i can't think of a better example that will definitely run... but here's a beefier hypothetical example
# to run the command: ./myProg input.txt output.txt --num_cats 5
# it would look like:
# command = "./myProg"
# args = ["input.txt", "output.txt", "--num_cats", "5"]
# or maybe this would also work
# args = ["input.txt", "output.txt", "--num_cats 5"]
# run another command with getProcessOutput [to capture the stdout output]
# or with getProcessValue [to just capture the exit code of the command]
out = yield utils.getProcessOutput(command, args)
val = yield utils.getProcessValue(command, args)
# printing is the same as logging
print "getProcessOutput:", out
print "getProcessValue:", val
# here's how you get stuff to display back in the web browser
request.write("getProcessOutput: %s" % out)
# you can do as many request.writes as you like, whenever/whereever
request.write("<br />getProcessValue: %d" % val)
# finish the request so the browser doesn't expect more
request.finish()
# how you finish up a method that has the @defer.inlineCallbacks decorator
defer.returnValue(None)
def _prepare_base_image(self):
"""
I am a private method for creating (possibly cheap) copies of a
base_image for start_instance to boot.
"""
if not self.base_image:
return defer.succeed(True)
if self.cheap_copy:
clone_cmd = "qemu-img"
clone_args = "create -b %(base)s -f qcow2 %(image)s"
else:
clone_cmd = "cp"
clone_args = "%(base)s %(image)s"
clone_args = clone_args % {
"base": self.base_image,
"image": self.image,
}
log.msg("Cloning base image: %s %s'" % (clone_cmd, clone_args))
d = utils.getProcessValue(clone_cmd, clone_args.split())
def _log_result(res):
log.msg("Cloning exit code was: %d" % res)
return res
def _log_error(err):
log.err("Cloning failed: %s" % err)
return err
d.addCallbacks(_log_result, _log_error)
return d
def _check(unit):
command = _HOST_COMMAND + [b"/usr/bin/systemctl", b"status"] + [unit]
Message.new(
system="log-agent:os-detection:journald",
unit=unit,
command=command,
).write()
return getProcessValue(command[0], command[1:], env=environ)
def testLog(self):
exes = which("buildbot")
if not exes:
raise unittest.SkipTest("Buildbot needs to be installed")
self.buildbotexe = exes[0]
d = getProcessValue(self.buildbotexe, ["create-master", "--log-size=1000", "--log-count=2", "master"])
d.addCallback(self._master_created)
return d
def testValue(self):
exe = sys.executable
scriptFile = self.makeSourceFile([
"import sys",
"sys.exit(1)"
])
d = utils.getProcessValue(exe, ['-u', scriptFile])
return d.addCallback(self.assertEquals, 1)
def _uncleanSocketTest(self, callback):
self.filename = self.mktemp()
source = ("from twisted.internet import protocol, reactor\n"
"reactor.listenUNIX(%r, protocol.ServerFactory(), wantPID=True)\n") % (self.filename,)
env = {'PYTHONPATH': os.pathsep.join(sys.path)}
d = utils.getProcessValue(sys.executable, ("-u", "-c", source), env=env)
d.addCallback(callback)
return d
def testValue(self):
exe = sys.executable
scriptFile = self.makeSourceFile([
"import sys",
"sys.exit(1)"
])
d = utils.getProcessValue(exe, ['-u', scriptFile])
return d.addCallback(self.assertEquals, 1)
def _do_exec(self, cmd, args, filename): if self.blocking: retval = call([cmd]+ args) return self._handle_process_run(retval, filename, args) else: d = utils.getProcessValue(cmd, args) d.addCallback(self._handle_process_run, filename, args) d.addErrback(self._handle_failure) return d
def done(self):
self.publish("Compressing")
try:
code = yield getProcessValue("/bin/tar", ["-cf", safe_format("./tmp/{}.tar", self.url), "-C", "./tmp/", self.url], os.environ, ".")
except:
self.publish("Failed to compress archive", True)
shutil.rmtree(self.folder)
else:
self.cleanup()
def test_value(self):
"""
The L{Deferred} returned by L{getProcessValue} is fired with the exit
status of the child process.
"""
scriptFile = self.makeSourceFile(["raise SystemExit(1)"])
d = utils.getProcessValue(self.exe, ['-u', scriptFile])
return d.addCallback(self.assertEqual, 1)
def _on_torrent_finished(self, torrent_id):
"""
This is called when a torrent finishes and checks if any files to extract.
"""
tid = component.get("TorrentManager").torrents[torrent_id]
tid_status = tid.get_status(["save_path", "move_completed", "name"])
if tid_status["move_completed"]:
log.warning(
"Cannot extract torrents with 'Move Completed' enabled")
return
files = tid.get_files()
for f in files:
file_root, file_ext = os.path.splitext(f["path"])
file_ext_sec = os.path.splitext(file_root)[1]
if file_ext_sec and file_ext_sec + file_ext in EXTRACT_COMMANDS:
file_ext = file_ext_sec + file_ext
elif file_ext not in EXTRACT_COMMANDS or file_ext_sec == '.tar':
log.warning("Can't extract file with unknown file type: %s" %
f["path"])
continue
cmd = EXTRACT_COMMANDS[file_ext]
# Now that we have the cmd, lets run it to extract the files
fpath = os.path.join(tid_status["save_path"],
os.path.normpath(f["path"]))
# Get the destination path
dest = os.path.normpath(self.config["extract_path"])
if self.config["use_name_folder"]:
name = tid_status["name"]
dest = os.path.join(dest, name)
# Create the destination folder if it doesn't exist
if not os.path.exists(dest):
try:
os.makedirs(dest)
except Exception, e:
log.error("Error creating destination folder: %s", e)
return
def on_extract_success(result, torrent_id, fpath):
# XXX: Emit an event
log.info("Extract successful: %s (%s)", fpath, torrent_id)
def on_extract_failed(result, torrent_id, fpath):
# XXX: Emit an event
log.error("Extract failed: %s (%s)", fpath, torrent_id)
# Run the command and add some callbacks
log.debug("Extracting %s with %s %s to %s", fpath, cmd[0], cmd[1],
dest)
d = getProcessValue(cmd[0], cmd[1].split() + [str(fpath)], {},
str(dest))
d.addCallback(on_extract_success, torrent_id, fpath)
d.addErrback(on_extract_failed, torrent_id, fpath)
def configure_iface(self, iface, ip='', action='up'):
"""See :meth:`wader.common.interfaces.IOSPlugin.configure_iface`"""
assert action in ['up', 'down']
if action == 'down':
args = [iface, action]
else:
args = [iface, ip, 'netmask', '255.255.255.255', '-arp', action]
return utils.getProcessValue('/sbin/ifconfig', args, reactor=reactor)
def _do_exec(self, cmd, args, filename):
if self.blocking:
retval = call([cmd] + args)
return self._handle_process_run(retval, filename, args)
else:
d = utils.getProcessValue(cmd, args)
d.addCallback(self._handle_process_run, filename, args)
d.addErrback(self._handle_failure)
return d
def test_value(self):
"""
The L{Deferred} returned by L{getProcessValue} is fired with the exit
status of the child process.
"""
scriptFile = self.makeSourceFile(["raise SystemExit(1)"])
d = utils.getProcessValue(self.exe, ['-u', scriptFile])
return d.addCallback(self.assertEqual, 1)
def ensureGitRepository(path=None, reactor=None):
if path is None:
path = os.getcwd()
d = getProcessValue('git', ('rev-parse', '--git-dir'), path=path, reactor=reactor)
def convertExitCode(res):
if res != 0:
raise NotAGitRepository
d.addCallback(convertExitCode)
return d
def testLog(self):
exes = which('buildbot')
if not exes:
raise unittest.SkipTest("Buildbot needs to be installed")
self.buildbotexe = exes[0]
d = getProcessValue(self.buildbotexe,
['create-master', '--log-size=1000', '--log-count=2',
'master'])
d.addCallback(self._master_created)
return d
def run(self):
"""
Executes _filepath and eventually calls _finish.
"""
if not self.version:
return self._finish(MISSING_VERSION)
if not self._filepath.exists():
return self._finish(FILE_NOT_FOUND)
d = utils.getProcessValue(self._filepath.path, args=(self.version,))
d.addCallback(self._finish)
def _optimize(self, optimize_dir):
# TODO: actually call the optimizer via twisted
# (reactor.callProcess or something, google knows)
optimizer_binary = self.run_config.optimizer_binary
deferred = getProcessValue(str(optimizer_binary),
[os.path.join(optimize_dir, "life"),
self.population_file])
deferred.addCallback(lambda x: self.load_genome())
def run_script(script):
result = getProcessValue(script, env=os.environ)
def validate_exit_code(code, script):
if code != 0:
raise CharmScriptError(script, code)
else:
return "%s succeeded." % script
return result.addCallback(validate_exit_code, script)
def resize(ignoredResult, img=img):
def getImageContents(result):
contents = open(img, 'rb').read()
os.unlink(img)
os.rmdir(tempdir)
return contents
d = utils.getProcessValue('/usr/bin/sips', ['sips', '-z', '240', '320',
'-s', 'formatOptions', '50%', img])
d.addCallback(getImageContents)
return d
def test_extendedAttributes(self):
"""
Test the return of extended attributes by the server: the sftp client
should ignore them, but still be able to parse the response correctly.
This test is mainly here to check that
L{filetransfer.FILEXFER_ATTR_EXTENDED} has the correct value.
"""
fn = self.mktemp()
with open(fn, 'w') as f:
f.write("ls .\nexit")
port = self.server.getHost().port
oldGetAttr = FileTransferForTestAvatar._getAttrs
def _getAttrs(self, s):
attrs = oldGetAttr(self, s)
attrs["ext_foo"] = "bar"
return attrs
self.patch(FileTransferForTestAvatar, "_getAttrs", _getAttrs)
self.server.factory.expectedLoseConnection = True
# PubkeyAcceptedKeyTypes does not exist prior to OpenSSH 7.0 so we
# first need to check if we can set it. If we can, -V will just print
# the version without doing anything else; if we can't, we will get a
# configuration error.
d = getProcessValue('ssh',
('-o', 'PubkeyAcceptedKeyTypes=ssh-dss', '-V'))
def hasPAKT(status):
if status == 0:
args = ('-o', 'PubkeyAcceptedKeyTypes=ssh-dss')
else:
args = ()
# Pass -F /dev/null to avoid the user's configuration file from
# being loaded, as it may contain settings that cause our tests to
# fail or hang.
args += ('-F', '/dev/null', '-o', 'IdentityFile=dsa_test', '-o',
'UserKnownHostsFile=kh_test', '-o',
'HostKeyAlgorithms=ssh-rsa', '-o', 'Port=%i' % (port, ),
'-b', fn, '[email protected]')
return args
def check(result):
self.assertEqual(result[2], 0)
for i in [
b'testDirectory', b'testRemoveFile', b'testRenameFile',
b'testfile1'
]:
self.assertIn(i, result[0])
d.addCallback(hasPAKT)
d.addCallback(lambda args: getProcessOutputAndValue('sftp', args))
return d.addCallback(check)
def _on_torrent_finished(self, torrent_id):
"""
This is called when a torrent finishes. We need to check to see if there
are any files to extract.
"""
# Get the save path
save_path = component.get("TorrentManager")[torrent_id].get_status(
["save_path"])["save_path"]
files = component.get("TorrentManager")[torrent_id].get_files()
for f in files:
ext = os.path.splitext(f["path"])
if ext[1] in (".gz", ".bz2", ".lzma"):
# We need to check if this is a tar
if os.path.splitext(ext[0]) == ".tar":
cmd = EXTRACT_COMMANDS[".tar" + ext[1]]
else:
if ext[1] in EXTRACT_COMMANDS:
cmd = EXTRACT_COMMANDS[ext[1]]
else:
log.debug("Can't extract unknown file type: %s", ext[1])
continue
# Now that we have the cmd, lets run it to extract the files
fp = os.path.join(save_path, f["path"])
# Get the destination path
dest = self.config["extract_path"]
if self.config["use_name_folder"]:
name = component.get("TorrentManager")[torrent_id].get_status(
["name"])["name"]
dest = os.path.join(dest, name)
# Create the destination folder if it doesn't exist
if not os.path.exists(dest):
try:
os.makedirs(dest)
except Exception, e:
log.error("Error creating destination folder: %s", e)
return
log.debug("Extracting to %s", dest)
def on_extract_success(result, torrent_id):
# XXX: Emit an event
log.debug("Extract was successful for %s", torrent_id)
def on_extract_failed(result, torrent_id):
# XXX: Emit an event
log.debug("Extract failed for %s", torrent_id)
# Run the command and add some callbacks
d = getProcessValue(cmd[0], cmd[1].split() + [str(fp)], {},
str(dest))
d.addCallback(on_extract_success, torrent_id)
d.addErrback(on_extract_failed, torrent_id)
def _on_torrent_finished(self, torrent_id):
"""
This is called when a torrent finishes and checks if any files to extract.
"""
tid = component.get("TorrentManager").torrents[torrent_id]
tid_status = tid.get_status(["save_path", "name"])
files = tid.get_files()
for f in files:
file_root, file_ext = os.path.splitext(f["path"])
file_ext_sec = os.path.splitext(file_root)[1]
if file_ext_sec and file_ext_sec + file_ext in EXTRACT_COMMANDS:
file_ext = file_ext_sec + file_ext
elif file_ext not in EXTRACT_COMMANDS or file_ext_sec == '.tar':
log.warning("EXTRACTOR: Can't extract file with unknown file type: %s" % f["path"])
continue
cmd = EXTRACT_COMMANDS[file_ext]
# Now that we have the cmd, lets run it to extract the files
fpath = os.path.join(tid_status["save_path"], os.path.normpath(f["path"]))
# Get the destination path
dest = os.path.normpath(self.config["extract_path"])
if self.config["use_name_folder"]:
name = tid_status["name"]
dest = os.path.join(dest, name)
# Override destination if in_place_extraction is set
if self.config["in_place_extraction"]:
name = tid_status["name"]
save_path = tid_status["save_path"]
dest = os.path.join(save_path,name)
# Create the destination folder if it doesn't exist
if not os.path.exists(dest):
try:
os.makedirs(dest)
except Exception, e:
log.error("EXTRACTOR: Error creating destination folder: %s", e)
return
def on_extract_success(result, torrent_id, fpath):
# XXX: Emit an event
log.info("EXTRACTOR: Extract successful: %s (%s)", fpath, torrent_id)
def on_extract_failed(result, torrent_id, fpath):
# XXX: Emit an event
log.error("EXTRACTOR: Extract failed: %s (%s)", fpath, torrent_id)
# Run the command and add some callbacks
log.debug("EXTRACTOR: Extracting fpath:%s with cmd[0]:%s cmd[1]:%s to dest: %s", fpath, cmd[0], cmd[1], dest)
d = getProcessValue(cmd[0], cmd[1].split() + [str(fpath)], {}, str(dest))
d.addCallback(on_extract_success, torrent_id, fpath)
d.addErrback(on_extract_failed, torrent_id, fpath)
def run():
env = dict(os.environ)
for (test, description) in TESTS:
if tests and test not in tests:
continue
print description
script = os.path.join(BASE, test + '.py')
yield utils.getProcessValue('python', [script], env)
result = raw_input("Did it work? [y/N] ")
reportFtest(test, result)
reactor.stop()
def execute(self, remoteCommand, process, sshArgs=""):
"""
Connects to the SSH server started in L{ConchServerSetupMixin.setUp} by
running the 'ssh' command line tool.
@type remoteCommand: str
@param remoteCommand: The command (with arguments) to run on the
remote end.
@type process: L{ConchTestOpenSSHProcess}
@type sshArgs: str
@param sshArgs: Arguments to pass to the 'ssh' process.
@return: L{defer.Deferred}
"""
# PubkeyAcceptedKeyTypes does not exist prior to OpenSSH 7.0 so we
# first need to check if we can set it. If we can, -V will just print
# the version without doing anything else; if we can't, we will get a
# configuration error.
d = getProcessValue(
which("ssh")[0], ("-o", "PubkeyAcceptedKeyTypes=ssh-dss", "-V"))
def hasPAKT(status):
if status == 0:
opts = "-oPubkeyAcceptedKeyTypes=ssh-dss "
else:
opts = ""
process.deferred = defer.Deferred()
# Pass -F /dev/null to avoid the user's configuration file from
# being loaded, as it may contain settings that cause our tests to
# fail or hang.
cmdline = ((
"ssh -2 -l testuser -p %i "
"-F /dev/null "
"-oUserKnownHostsFile=kh_test "
"-oPasswordAuthentication=no "
# Always use the RSA key, since that's the one in kh_test.
"-oHostKeyAlgorithms=ssh-rsa "
"-a "
"-i dsa_test ") + opts + sshArgs + " 127.0.0.1 " +
remoteCommand)
port = self.conchServer.getHost().port
cmds = (cmdline % port).split()
encodedCmds = []
for cmd in cmds:
if isinstance(cmd, str):
cmd = cmd.encode("utf-8")
encodedCmds.append(cmd)
reactor.spawnProcess(process, which("ssh")[0], encodedCmds)
return process.deferred
return d.addCallback(hasPAKT)
def _on_torrent_finished(self, torrent_id):
"""
This is called when a torrent finishes and checks if any files to extract.
"""
tid = component.get("TorrentManager").torrents[torrent_id]
tid_status = tid.get_status(["save_path", "move_completed", "name"])
if tid_status["move_completed"]:
log.error("EXTRACTOR: Cannot extract torrents with 'Move Completed' enabled")
return
files = tid.get_files()
for f in files:
cmd = ''
#file_ext = os.path.splitext(os.path.splitext(f["path"])[0])[1] + os.path.splitext(f["path"])[1]
file_ext = '.' + f['path'].split('.')[-1]
if file_ext in EXTRACT_COMMANDS:
cmd = EXTRACT_COMMANDS[file_ext]
else:
log.error("EXTRACTOR: Can't extract unknown file type: %s", file_ext)
continue
# Now that we have the cmd, lets run it to extract the files
fpath = os.path.join(tid_status["save_path"], os.path.normpath(f["path"]))
# Get the destination path
dest = os.path.normpath(self.config["extract_path"])
if self.config["use_name_folder"]:
name = tid_status["name"]
dest = os.path.join(dest, name)
# Create the destination folder if it doesn't exist
if not os.path.exists(dest):
try:
os.makedirs(dest)
except Exception, e:
log.error("EXTRACTOR: Error creating destination folder: %s", e)
return
def on_extract_success(result, torrent_id, fpath):
# XXX: Emit an event
log.info("EXTRACTOR: Extract successful: %s (%s)", fpath, torrent_id)
def on_extract_failed(result, torrent_id, fpath):
# XXX: Emit an event
log.error("EXTRACTOR: Extract failed: %s (%s)", fpath, torrent_id)
# Run the command and add some callbacks
if cmd:
log.debug("EXTRACTOR: Extracting %s with %s %s to %s", fpath, cmd[0], cmd[1], dest)
d = getProcessValue(cmd[0], cmd[1].split() + [str(fpath)], {}, str(dest))
d.addCallback(on_extract_success, torrent_id, fpath)
d.addErrback(on_extract_failed, torrent_id, fpath)
def _uncleanSocketTest(self, callback):
self.filename = self.mktemp()
source = (
"from twisted.internet import protocol, reactor\n"
"reactor.listenUNIX(%r, protocol.ServerFactory(), wantPID=True)\n"
) % (self.filename, )
env = {'PYTHONPATH': os.pathsep.join(sys.path)}
d = utils.getProcessValue(sys.executable, ("-u", "-c", source),
env=env)
d.addCallback(callback)
return d
def _uncleanSocketTest(self, callback):
self.filename = self.mktemp()
source = networkString((
"from twisted.internet import protocol, reactor\n"
"reactor.listenUNIX(%r, protocol.ServerFactory(),"
"wantPID=True)\n") % (self.filename,))
env = {b'PYTHONPATH': FilePath(
os.pathsep.join(sys.path)).asBytesMode().path}
pyExe = FilePath(sys.executable).asBytesMode().path
d = utils.getProcessValue(pyExe, (b"-u", b"-c", source), env=env)
d.addCallback(callback)
return d
def ensureGitRepository(path=None, reactor=None):
if path is None:
path = os.getcwd()
d = getProcessValue('git', ('rev-parse', '--git-dir'),
path=path,
reactor=reactor)
def convertExitCode(res):
if res != 0:
raise NotAGitRepository
d.addCallback(convertExitCode)
return d
def test_pack_sdist(self):
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
with self._generate_docs():
self.config.parseOptions()
self.config['name'] = 'mamba-dummy'
self.packer.pack_application(
'sdist', self.config,
config.Application('config/application.json'))
self.assertTrue(os.path.exists('mamba-dummy-0.1.2.tar.gz'))
self.packer.do(['rm', 'mamba-dummy-0.1.2.tar.gz'])
def run(self, item):
if len(self.running) >= self.running_max:
return False
env = {
'LD_LIBRARY_PATH': '/usr/local/lib',
'LANG': 'C.UTF-8',
}
args = [
'/opt/xmppoke/xmppoke.lua',
'--cafile=/etc/ssl/certs/ca-certificates.crt',
'--mode=' + item.mode.encode('utf-8'),
'-d=15',
'-v',
item.domain.encode('utf-8'),
]
for arg, value in self.argv.items():
if not value:
continue
args.append(arg + "=" + value)
logmsg(u"starting %s" % (item))
try:
self.running.append(item)
utils.getProcessValue('luajit', args, env,
path="/opt/xmppoke").addCallbacks(
callback=self.child_done,
callbackArgs=(item, ),
errback=self.child_failed,
errbackArgs=(item, ),
)
except Exception, e:
# as a precaution, remove all similar items from the queue
self.queue.remove(item)
self.child_failed("failed to execute %r: %s" % (args, e), item)
return False
def test_sql_dump_to_file(self):
#_check_mamba_admin_tool()
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
with fake_project():
yield utils.getProcessOutput('mamba-admin',
['sql', 'dump', 'test'], os.environ)
dump_file = filepath.FilePath('test.sql')
self.assertTrue(dump_file.exists())
self.assertTrue(dump_file.getsize() > 0)
dump_file.remove()
def run(self):
"""
Run the configured program and call the update() method with the exit status.
"""
if not self.program:
self.update(-1)
return
use_shell = not (isinstance(self.program, (list, tuple)))
logging.debug('Calling %s',
self.program if use_shell else ' '.join(self.program))
d = utils.getProcessValue(
self.program[0],
self.program[1:] if len(self.program) > 1 else [],
env=os.environ)
d.addCallback(self.update)
def _uncleanSocketTest(self, callback):
self.filename = self.mktemp()
source = networkString(
("from twisted.internet import protocol, reactor\n"
"reactor.listenUNIX(%r, protocol.ServerFactory(),"
"wantPID=True)\n") % (self.filename, ))
env = {
b'PYTHONPATH':
FilePath(os.pathsep.join(sys.path)).asBytesMode().path
}
pyExe = FilePath(sys.executable).asBytesMode().path
d = utils.getProcessValue(pyExe, (b"-u", b"-c", source), env=env)
d.addCallback(callback)
return d
def _testHostAgainstRelease(self, host, release):
"""
Tests a given host against a specific certifi release. Returns a
Deferred that fires with the result of the test.
"""
log = self._log.bind(function="_testHostAgainstRelease")
file_path = os.path.join(self.certifiDirectory, 'certifi-' + release)
args = [self.binPath, file_path, host]
log.msg("running test", args=args)
result = yield getProcessValue(
distutils.spawn.find_executable('pypy'),
args=args,
)
returnValue(result)
def test_isGitRepository(self):
"""
When called from a git repository, L{git.ensureGitRepository} returns
a deferred that doesn't errback.
"""
basedir = FilePath(self.mktemp())
basedir.createDirectory()
basedir.child('.git').setContent('blah-blah-blah')
gitRepo = basedir.child('git-repo')
# Create a git repository
d = getProcessValue('git', ('init', gitRepo.path))
d.addCallback(lambda _: git.ensureGitRepository(gitRepo.path))
return d
def stop_instance(self, fast=False):
log.msg("Attempting to stop '%s'" % self.name)
retval = yield utils.getProcessValue(self.stop_script[0],
self.stop_script[1:])
log.msg("slave destroyed (%s): Forcing its connection closed." %
self.name)
yield AbstractBuildSlave.disconnect(self)
log.msg(
"We forced disconnection (%s), cleaning up and triggering new build"
% self.name)
self.botmaster.maybeStartBuildsForSlave(self.name)
defer.returnValue(retval == 0)
def execute(self, remoteCommand, process, sshArgs=''):
"""
Connects to the SSH server started in L{ConchServerSetupMixin.setUp} by
running the 'ssh' command line tool.
@type remoteCommand: str
@param remoteCommand: The command (with arguments) to run on the
remote end.
@type process: L{ConchTestOpenSSHProcess}
@type sshArgs: str
@param sshArgs: Arguments to pass to the 'ssh' process.
@return: L{defer.Deferred}
"""
# PubkeyAcceptedKeyTypes does not exist prior to OpenSSH 7.0 so we
# first need to check if we can set it. If we can, -V will just print
# the version without doing anything else; if we can't, we will get a
# configuration error.
d = getProcessValue('ssh',
('-o', 'PubkeyAcceptedKeyTypes=ssh-dss', '-V'))
def hasPAKT(status):
if status == 0:
opts = '-oPubkeyAcceptedKeyTypes=ssh-dss '
else:
opts = ''
process.deferred = defer.Deferred()
# Pass -F /dev/null to avoid the user's configuration file from
# being loaded, as it may contain settings that cause our tests to
# fail or hang.
cmdline = ('ssh -2 -l testuser -p %i '
'-F /dev/null '
'-oUserKnownHostsFile=kh_test '
'-oPasswordAuthentication=no '
# Always use the RSA key, since that's the one in kh_test.
'-oHostKeyAlgorithms=ssh-rsa '
'-a '
'-i dsa_test ') + opts + sshArgs + \
' 127.0.0.1 ' + remoteCommand
port = self.conchServer.getHost().port
cmds = (cmdline % port).split()
reactor.spawnProcess(process, "ssh", cmds)
return process.deferred
return d.addCallback(hasPAKT)
def test_sql_dump(self):
result = yield utils.getProcessValue('mamba-admin', [], os.environ)
if result == 1:
raise unittest.SkipTest('mamba framework is not installed yet')
with fake_project():
result = yield utils.getProcessOutput('mamba-admin',
['sql', 'dump'], os.environ)
print(result)
self.assertTrue("INSERT INTO 'dummy'" in result)
self.assertTrue("INSERT INTO 'stubing'" in result)
self.assertTrue('Test row 1' in result)
self.assertTrue('Test row 2' in result)
self.assertTrue('Test row 3' in result)
