def script(self):
jdk = self.options["JavaHome"]
if jdk is None:
if "JAVA_HOME" in os.environ:
jdk = os.environ["JAVA_HOME"]
else:
stat, jdk, _err = li.mycmd("readlink -f $(which java)")
jdk = jdk.strip()
if stat or not jdk.endswith("/jre/bin/java"):
print(
"Warning: could not detect JAVA version, probably you don't have a local hadoop client, "
"so I'm skipping hadoop python libraries, try setting JAVA_HOME manually"
)
jdk = None
else:
jdk = jdk[:-len("/jre/bin/java")]
hdh = self.options["HadoopHome"]
if hdh is None:
if "HADOOP_HOME" in os.environ:
hdh = os.environ["HADOOP_HOME"]
else:
stat, hdh, _err = li.mycmd(" readlink -f $(which hadoop)")
hdh = hdh.strip()
if stat or not hdh.endswith("/bin/hadoop"):
print(
"INFO: could not detect hadoop installation, probably you don't have a local hadoop "
"client, so I'm skipping hadoop python libraries, try setting HADOOP_HOME manually"
)
hdh = None
else:
hdh = hdh[:-len("/bin/hadoop")]
if hdh is not None and jdk is not None:
# find hadoop version ...
stat, hdv, _err = li.mycmd("hadoop version")
hdv = '.'.join([l for l in hdv.split('\n')
if "Hadoop" in l][0].split(" ")[-1].split('.')[:3])
for ezmodule in self.options["easy_install"]:
self.run(
"bash -c 'source " + self.toolbox.envscript() +
" > /dev/null ; export HADOOP_VERSION=" + hdv +
"; export JAVA_HOME=" + jdk + "; export HADOOP_HOME=" +
hdh +
"; export CLASSPATH=$CLASSPATH:`hadoop classpath`; easy_install "
+ ezmodule + "'")
for pipmodule in self.options["pip"]:
sub.call(
"bash -c 'source " + self.toolbox.envscript() +
" > /dev/null ; export HADOOP_VERSION=" + hdv +
"; export JAVA_HOME=" + jdk + "; export HADOOP_HOME=" +
hdh +
"; export CLASSPATH=$CLASSPATH:`hadoop classpath`; pip install "
+ pipmodule + "'",
shell=True)
return
def script(self):
jdk = self.options["JavaHome"]
if jdk is None:
if "JAVA_HOME" in os.environ:
jdk = os.environ["JAVA_HOME"]
else:
stat, jdk, err = li.mycmd("readlink -f $(which java)")
jdk = jdk.strip()
if stat or not jdk.endswith("/jre/bin/java"):
print "Warning: could not detect JAVA version, probably you don't have a local hadoop client, " \
"so I'm skipping hadoop python libraries, try setting JAVA_HOME manually"
jdk = None
else:
jdk = jdk[:-len("/jre/bin/java")]
hdh = self.options["HadoopHome"]
if hdh is None:
if "HADOOP_HOME" in os.environ:
hdh = os.environ["HADOOP_HOME"]
else:
stat, hdh, err = li.mycmd(" readlink -f $(which hadoop)")
hdh = hdh.strip()
if stat or not hdh.endswith("/bin/hadoop"):
print "INFO: could not detect hadoop installation, probably you don't have a local hadoop " \
"client, so I'm skipping hadoop python libraries, try setting HADOOP_HOME manually"
hdh = None
else:
hdh = hdh[:-len("/bin/hadoop")]
if hdh is not None and jdk is not None:
# find hadoop version ...
stat, hdv, err = li.mycmd("hadoop version")
hdv = '.'.join([l for l in hdv.split('\n') if "Hadoop" in l][0].split(" ")[-1].split('.')[:3])
for ezmodule in self.options["easy_install"]:
self.run(
"bash -c 'source " + self.toolbox.envscript() + " > /dev/null ; export HADOOP_VERSION=" + hdv +
"; export JAVA_HOME=" + jdk + "; export HADOOP_HOME=" + hdh
+ "; export CLASSPATH=$CLASSPATH:`hadoop classpath`; easy_install " +
ezmodule + "'")
for pipmodule in self.options["pip"]:
self.run(
"bash -c 'source " + self.toolbox.envscript() + " > /dev/null ; export HADOOP_VERSION=" + hdv +
"; export JAVA_HOME=" + jdk + "; export HADOOP_HOME=" + hdh
+ "; export CLASSPATH=$CLASSPATH:`hadoop classpath`; pip install " +
pipmodule + "'")
return
def runTest(self):
"""
Check the functionality of each tiny helper function in the kaveinstall library
"""
import kaveinstall as ki
self.assertEqual(ki.repoURL('fn', 'repo', 'arch', 'dir',
'ver'), 'repo/arch/dir/ver/fn',
'unexpected output from RepoURL')
self.assertEqual(ki.mycmd('echo "w00t"'), (0, b'w00t\n', b''),
'unexpected output from mycmd')
self.assertRaises(RuntimeError, ki.throw_on_fail_quiet, 'exit 1')
self.assertRaises(RuntimeError, ki.throw_on_fail_loud, 'exit 1')
import tempfile
tdir = tempfile.mkdtemp()
self.assertRaises(RuntimeError, ki.clean_on_fail_quiet, 'exit 1', tdir)
self.assertFalse(os.path.exists(tdir),
'cleaning (quiet) failed to work')
tdir = tempfile.mkdtemp()
self.assertRaises(RuntimeError, ki.clean_on_fail_loud, 'exit 1', tdir)
self.assertFalse(os.path.exists(tdir),
'cleaning (loud) failed to work')
self.assertTrue(ki.detect_linux_version() in ["Centos7", "Ubuntu16"],
'Unexpected OS result!')
self.assertTrue(
len(ki.df('/')) == 6, 'df -P returned strange results!')
prot = {
"http:": "wget",
"https:": "wget",
"ftp:": "wget",
"/tmp": "cp"
}
for p, m in prot.items():
self.assertTrue(
ki.copymethods(p, 'blah').startswith(m),
m + ' copymethod not used for ' + p)
self.assertRaises(IOError, ki.failoversources, [
"https://thisisnotawebsite1928918318971736661181818187222"
".kadklhasdaiuqiowehasdhawq.blah/"
])
self.assertEqual(
ki.failoversources([
"https://thisisnotawebsite1928918318971736661181818187222"
".kadklhasdaiuqiowehasdhawq.blah/", "http://google.com"
]), "http://google.com")
nn = None
with open(os.devnull, 'w') as devnull:
with base.RedirectStdOut(devnull):
nn = ki.fromKPMGrepo('notafilenotafilenotafilenot')
self.assertTrue(nn is None, "Expected failure of repo file not seen")
self.assertTrue(
ki.fromKPMGrepo('') is not None,
"Default repository not accessible")
def runTest(self):
"""
Runs the predefined tests on one installed piece of software
Each component has a tests list of tuples, component.tests
these look like: ('command',errcode,'stdout','stderr')
Tests are always run in the Kave environment
The command can have the %%INSTALLDIR%%,%%INSTALLDIRPRO%%,%%INSTALLDIRVERSION%%
directives for search/replace
"""
self.component.constinstdir()
# Deal with mixed 2.X / 1.X versions!
if self.component.cname not in ["KaveToolbox"]:
# this means the component is installed but a different version (testable)
if not os.path.exists(self.component.installDirVersion):
self.component.installDirVersion = self.component.installDirPro
# this means the KaveToolbox is still 1.X
if not os.path.exists(self.component.toolbox.installDirVersion):
self.component.installDirVersion = self.component.installDirPro
# this means a 1.X version of the component is installed
if not os.path.exists(self.component.installDirPro):
self.component.installDirVersion = self.component.installDir
self.component.installDirPro = self.component.installDir
# Now, run the tests if this component was supposed to be installed.
if getattr(self.component, kind) and self.component.doInstall:
self.component.loud = False
for ttuple in self.component.tests:
cmd, rc, sin, sout = ttuple
ttuple = (
self.component.knownreplaces(cmd),
rc,
self.component.knownreplaces(sin).encode('utf-8'),
self.component.knownreplaces(sout).encode('utf-8')
)
cmd = ttuple[0]
script = self.component.toolbox.envscript()
if os.path.exists(script.replace('/pro/', ki.__version__)):
script = script.replace('/pro/', ki.__version__) + " " + ki.__version__
newtuple = ki.mycmd("bash -c 'source "
+ script
+ " > /dev/null ;" + cmd + ";'")
self.assertEquals(ttuple[1:], newtuple, self.component.cname
+ ": Unexpected failure with component " + self.component.cname
+ ": \n - I was expecting:\n\t" + cmd + ttuple[1:].__str__()
+ "\n - But I received:\n\t " + newtuple.__str__())
def runTest(self):
"""
Runs the predefined tests on one installed piece of software
Each component has a tests list of tuples, component.tests
these look like: ('command',errcode,'stdout','stderr')
Tests are always run in the Kave environment
The command can have the %%INSTALLDIR%%,%%INSTALLDIRPRO%%,%%INSTALLDIRVERSION%%
directives for search/replace
"""
self.component.constinstdir()
# Deal with mixed 2.X / 1.X versions!
if self.component.cname not in ["KaveToolbox"]:
# this means the component is installed but a different version (testable)
if not os.path.exists(self.component.installDirVersion):
self.component.installDirVersion = self.component.installDirPro
# this means the KaveToolbox is still 1.X
if not os.path.exists(self.component.toolbox.installDirVersion):
self.component.installDirVersion = self.component.installDirPro
# this means a 1.X version of the component is installed
if not os.path.exists(self.component.installDirPro):
self.component.installDirVersion = self.component.installDir
self.component.installDirPro = self.component.installDir
# Now, run the tests if this component was supposed to be installed.
if getattr(self.component, kind) and self.component.doInstall:
self.component.loud = False
for ttuple in self.component.tests:
cmd, rc, sin, sout = ttuple
ttuple = (self.component.knownreplaces(cmd), rc,
self.component.knownreplaces(sin).encode('utf-8'),
self.component.knownreplaces(sout).encode('utf-8'))
cmd = ttuple[0]
script = self.component.toolbox.envscript()
if os.path.exists(script.replace('/pro/', ki.__version__)):
script = script.replace(
'/pro/', ki.__version__) + " " + ki.__version__
newtuple = ki.mycmd("bash -c 'source " + script +
" > /dev/null ;" + cmd + ";'")
self.assertEquals(
ttuple[1:], newtuple, self.component.cname +
": Unexpected failure with component " +
self.component.cname + ": \n - I was expecting:\n\t" +
cmd + ttuple[1:].__str__() + "\n - But I received:\n\t " +
newtuple.__str__())
def runTest(self):
"""
Check the functionality of each tiny helper function in the kaveinstall library
"""
import kaveinstall as ki
self.assertEqual(ki.repoURL('fn', 'repo', 'arch', 'dir', 'ver'),
'repo/arch/dir/ver/fn', 'unexpected output from RepoURL')
self.assertEqual(ki.mycmd('echo "w00t"'), (0, b'w00t\n', b''),
'unexpected output from mycmd')
self.assertRaises(RuntimeError, ki.throw_on_fail_quiet, 'exit 1')
self.assertRaises(RuntimeError, ki.throw_on_fail_loud, 'exit 1')
import tempfile
tdir = tempfile.mkdtemp()
self.assertRaises(RuntimeError, ki.clean_on_fail_quiet, 'exit 1', tdir)
self.assertFalse(os.path.exists(tdir), 'cleaning (quiet) failed to work')
tdir = tempfile.mkdtemp()
self.assertRaises(RuntimeError, ki.clean_on_fail_loud, 'exit 1', tdir)
self.assertFalse(os.path.exists(tdir), 'cleaning (loud) failed to work')
self.assertTrue(ki.detect_linux_version() in ["Centos7", "Ubuntu16"],
'Unexpected OS result!')
self.assertTrue(len(ki.df('/')) == 6, 'df -P returned strange results!')
prot = {"http:": "wget", "https:": "wget", "ftp:": "wget", "/tmp": "cp"}
for p, m in prot.items():
self.assertTrue(ki.copymethods(p, 'blah').startswith(m), m + ' copymethod not used for ' + p)
self.assertRaises(IOError,
ki.failoversources,
["https://thisisnotawebsite1928918318971736661181818187222"
".kadklhasdaiuqiowehasdhawq.blah/"]
)
self.assertEqual(ki.failoversources(["https://thisisnotawebsite1928918318971736661181818187222"
".kadklhasdaiuqiowehasdhawq.blah/",
"http://google.com"]), "http://google.com")
nn = None
with open(os.devnull, 'w') as devnull:
with base.RedirectStdOut(devnull):
nn = ki.fromKPMGrepo('notafilenotafilenotafilenot')
self.assertTrue(nn is None,
"Expected failure of repo file not seen")
self.assertTrue(ki.fromKPMGrepo('') is not None,
"Default repository not accessible")
def skipif(self):
return (conda.installDirVersion in
mycmd("bash -c 'source " + self.toolbox.envscript()
+ " > /dev/null ; which root ;'")[1]
)
def skipif(self):
return (conda.installDirVersion
in mycmd("bash -c 'source " + self.toolbox.envscript() +
" > /dev/null ; which root ;'")[1])
def skipif(self):
return (conda.installDirVersion in
mycmd("bash -c 'source " + self.toolbox.envscript()
+ " > /dev/null ; python -c \"import rpy2; print(rpy2.__file__);\" ;'")[1]
)
def skipif(self):
return (conda.installDirVersion in
mycmd("bash -c 'source " + self.toolbox.envscript()
+ " > /dev/null ; python -c \"import pygsl; print(pygsl.__file__);\" ;'")[1]
)