def testRemoteTaskFromJupyter(self):
'''Test the execution of tasks with -q '''
from ipykernel.tests.utils import execute
from sos_notebook.test_utils import sos_kernel, get_display_data
subprocess.call(['sos', 'purge'])
with sos_kernel() as kc:
# the cell will actually be executed several times
# with automatic-reexecution
code = """
%run -s force -W -q ts -c ~/docker.yml
[10]
input: for_each={'i': range(2)}
task:
run: expand=True
echo this is jupyter pending test "{i}"
sleep {10+i}
"""
# these should be automatically rerun by the frontend
execute(kc=kc, code=code)
res = get_display_data(kc.iopub_channel, 'text/html')
# check for task?
execute(kc=kc, code='%tasks -q ts')
res = get_display_data(kc.iopub_channel, 'text/html')
self.assertTrue('table_ts_' in res, 'Got {}'.format(res))
# get IDs
# table_localhost_ac755352394584f797cebddf2c0b8ca7"
tid = res.split('table_ts_')[-1].split('"')[0]
# now we have the tid, we can check task info
execute(kc=kc, code='%taskinfo -q ts ' + tid)
res = get_display_data(kc.iopub_channel, 'text/html')
self.assertTrue(tid in res, 'expect {} in {}'.format(tid, res))
# there should be two tasks
lines = subprocess.check_output(['sos', 'status', '-q', 'ts', '-c', '~/docker.yml']).decode().splitlines()
self.assertGreaterEqual(len(lines), 2)
def testMagicUse(self):
with sos_kernel() as kc:
iopub = kc.iopub_channel
execute(kc=kc, code="%use R0 -l sos_r.kernel:sos_R -c #CCCCCC")
_, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
execute(kc=kc,
code="%use R1 -l sos_r.kernel:sos_R -k ir -c #CCCCCC")
_, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
execute(kc=kc, code="%use R2 -k ir")
_, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
execute(kc=kc, code="a <- 1024")
wait_for_idle(kc)
execute(kc=kc, code="a")
res = get_display_data(iopub)
self.assertEqual(res, '[1] 1024')
execute(kc=kc, code="%use R3 -k ir -l R")
_, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
execute(kc=kc, code="a <- 233")
wait_for_idle(kc)
execute(kc=kc, code="a")
res = get_display_data(iopub)
self.assertEqual(res, '[1] 233')
execute(kc=kc, code="%use R2 -c red")
_, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
execute(kc=kc, code="a")
res = get_display_data(iopub)
self.assertEqual(res, '[1] 1024')
execute(kc=kc, code="%use sos")
wait_for_idle(kc)
def testGetPythonDataFrameFromRuby(self):
# Python -> Ruby
with sos_kernel() as kc:
iopub = kc.iopub_channel
# create a data frame
execute(kc=kc, code='''
import pandas as pd
import numpy as np
arr = np.random.randn(1000)
arr[::10] = np.nan
df = pd.DataFrame({'column_{0}'.format(i): arr for i in range(10)})
''')
clear_channels(iopub)
execute(kc=kc, code="%use Ruby")
_, stderr = get_std_output(iopub)
self.assertEqual(stderr, '', "GOT ERROR {}".format(stderr))
execute(kc=kc, code="%get df")
wait_for_idle(kc)
execute(kc=kc, code="df.size()")
res = get_display_data(iopub)
self.assertEqual(res, '1000')
execute(kc=kc, code="df.vectors().to_a().size()")
res = get_display_data(iopub)
self.assertEqual(res, '10')
execute(kc=kc, code="%use sos")
wait_for_idle(kc)
def testPendingTask(self):
'''Test the execution of tasks with -s force'''
with sos_kernel() as kc:
# the cell will actually be executed several times
# with automatic-reexecution
code = """
%run -s force -W
[10]
input: for_each={'i': range(2)}
task:
run: expand=True
echo this is jupyter pending test "{i}"
sleep {10+i}
"""
# these should be automatically rerun by the frontend
execute(kc=kc, code=code)
wait_for_idle(kc)
# check for task?
execute(kc=kc, code='%tasks')
res = get_display_data(kc.iopub_channel, 'text/html')
# get IDs
# table_localhost_ac755352394584f797cebddf2c0b8ca7"
tid = res.split('table_localhost_')[-1].split('"')[0]
# now we have the tid, we can check task info
execute(kc=kc, code='%taskinfo ' + tid)
res = get_display_data(kc.iopub_channel, 'text/html')
self.assertTrue(tid in res)
def testMagicPreview(self):
with sos_kernel() as kc:
iopub = kc.iopub_channel
#
execute(kc=kc, code='''
%preview mtcars
%use R
''')
wait_for_idle(kc)
execute(kc=kc, code='''%use sos''')
wait_for_idle(kc)
# preview figure
execute(kc=kc,
code='''\
%preview -n a.png
R:
png('a.png')
plot(0)
dev.off()
''')
res = get_display_data(iopub, 'image/png')
self.assertGreater(len(res), 1000, 'Expect a image {}'.format(res))
# preview jpg
execute(kc=kc,
code='''\
%preview a.jp*
R:
jpeg('a.jpg')
plot(0)
dev.off()
''')
res = get_display_data(iopub, 'image/jpeg')
self.assertGreater(len(res), 1000, 'Expect a image {}'.format(res))
# preview pdf in iframe (by default)
execute(kc=kc,
code='''
%preview a.pdf
R:
pdf('a.pdf')
plot(0)
dev.off()
''')
# or png (which requires imagemagick
wait_for_idle(kc)
execute(kc=kc, code='''\
%preview a.pdf -s png
''')
# could return html or image depending on configuration
res = get_display_data(iopub, ('text/html', 'image/png'))
self.assertTrue('iframe' in res or len(res) > 1000,
'Expect a image {}'.format(res))
#
# switch back
execute(kc=kc, code='%use SoS')
wait_for_idle(kc)
def testAutoSharedVars(self):
with sos_kernel() as kc:
iopub = kc.iopub_channel
execute(kc=kc, code="sos_null = None")
wait_for_idle(kc)
execute(kc=kc, code="sos_num = 123")
wait_for_idle(kc)
execute(kc=kc, code="%use R")
wait_for_idle(kc)
execute(kc=kc, code="sos_num")
res = get_display_data(iopub)
self.assertEqual(res, '[1] 123')
execute(kc=kc, code="sos_num = sos_num + 10")
wait_for_idle(kc)
execute(kc=kc, code="%use sos")
wait_for_idle(kc)
execute(kc=kc, code="sos_num")
res = get_display_data(iopub)
self.assertEqual(res, '133')
def testMagicGet(self):
with sos_kernel() as kc:
iopub = kc.iopub_channel
execute(kc=kc, code="a = 1025")
wait_for_idle(kc)
execute(kc=kc, code="_b_a = 22")
wait_for_idle(kc)
execute(kc=kc, code="%use R")
_, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
execute(kc=kc, code="%get a")
wait_for_idle(kc)
execute(kc=kc, code="a")
res = get_display_data(iopub)
self.assertEqual(res, '[1] 1025')
execute(kc=kc, code="b <- 122\nc<-555")
wait_for_idle(kc)
#
execute(kc=kc, code="%get _b_a")
wait_for_idle(kc)
execute(kc=kc, code=".b_a")
res = get_display_data(iopub)
self.assertEqual(res, '[1] 22')
execute(kc=kc, code="%use sos")
wait_for_idle(kc)
#
# get from a sub kernel
execute(kc=kc, code="%get --from R b")
wait_for_idle(kc)
execute(kc=kc, code="b")
res = get_result(iopub)
self.assertEqual(res, 122)
# get from a third kernel
execute(kc=kc, code="%use Python3")
wait_for_idle(kc)
execute(kc=kc, code="%get --from R c")
wait_for_idle(kc)
execute(kc=kc, code="c")
res = get_result(iopub)
self.assertEqual(res, 555)
execute(kc=kc, code="%use sos")
wait_for_idle(kc)
def testAutoSharedVars(self):
'''Test the sharing of variables automatically'''
with sos_kernel() as kc:
iopub = kc.iopub_channel
execute(kc=kc, code="""\
sos_null = None
sos_num = 123
""")
wait_for_idle(kc)
execute(kc=kc, code="%use Python3")
wait_for_idle(kc)
execute(kc=kc, code="sos_num")
res = get_display_data(iopub)
self.assertEqual(res, '123')
execute(kc=kc, code="sos_num = sos_num + 10")
wait_for_idle(kc)
execute(kc=kc, code="%use sos")
wait_for_idle(kc)
execute(kc=kc, code="sos_num")
res = get_display_data(iopub)
self.assertEqual(res, '133')
def testSubKernel(self):
with sos_kernel() as kc:
iopub = kc.iopub_channel
execute(kc=kc, code="%use R")
_, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
execute(kc=kc, code="a <- 1024")
wait_for_idle(kc)
execute(kc=kc, code="a")
res = get_display_data(iopub)
self.assertEqual(res, '[1] 1024')
execute(kc=kc, code="%use sos")
wait_for_idle(kc)
def testVisualizer(self):
with sos_kernel() as kc:
# preview variable
iopub = kc.iopub_channel
# preview dataframe
execute(kc=kc,
code='''\
%preview mtcars -n -l 10
%get mtcars --from R
''')
res = get_display_data(iopub, 'text/html')
self.assertTrue('dataframe_container' in res and 'Mazda' in res,
'Expect preview {}'.format(res))
#
execute(kc=kc,
code='''
%preview mtcars -n -s scatterplot mpg disp --by cyl
%get mtcars --from R
''')
res = get_display_data(iopub, 'text/html')
self.assertTrue('Hornet' in res, 'Expect preview {}'.format(res))
#
execute(kc=kc,
code='''
%preview mtcars -n -s scatterplot _index disp hp mpg --tooltip wt qsec
%get mtcars --from R
''')
res = get_display_data(iopub, 'text/html')
self.assertTrue('Hornet' in res, 'Expect preview {}'.format(res))
#
execute(kc=kc,
code='''
%preview mtcars -n -s scatterplot disp hp --log xy --xlim 60 80 --ylim 40 300
%get mtcars --from R
''')
res = get_display_data(iopub, 'text/html')
self.assertTrue('Hornet' in res, 'Expect preview {}'.format(res))
def testGetPythonNoneFromRuby(self):
# Python -> Ruby
with sos_kernel() as kc:
iopub = kc.iopub_channel
# create a data frame
execute(kc=kc, code='''
null_var = None
''')
clear_channels(iopub)
execute(kc=kc, code="%use Ruby")
wait_for_idle(kc)
execute(kc=kc, code="%get null_var")
wait_for_idle(kc)
execute(kc=kc, code="null_var == nil")
res = get_display_data(iopub)
self.assertEqual(res, 'true')
execute(kc=kc, code="%use sos")
wait_for_idle(kc)
def testGetPythonMatrixFromRuby(self):
# Python -> Ruby
with sos_kernel() as kc:
iopub = kc.iopub_channel
# create a matrix
execute(kc=kc, code='''
import numpy as np
mat_var = np.matrix([[1,2],[3,4]])
''')
clear_channels(iopub)
execute(kc=kc, code="%use Ruby")
wait_for_idle(kc)
execute(kc=kc, code="%get mat_var")
wait_for_idle(kc)
execute(kc=kc, code="mat_var.size()")
res = get_display_data(iopub)
self.assertEqual(res, '4')
execute(kc=kc, code="%use sos")
wait_for_idle(kc)
def testGetPythonDataFrameFromR(self):
# Python -> R
with sos_kernel() as kc:
iopub = kc.iopub_channel
# create a data frame
execute(kc=kc,
code='''
import pandas as pd
import numpy as np
arr = np.random.randn(1000)
arr[::10] = np.nan
df = pd.DataFrame({'column_{0}'.format(i): arr for i in range(10)})
''')
clear_channels(iopub)
execute(kc=kc, code="%use R")
wait_for_idle(kc)
execute(kc=kc, code="%get df")
wait_for_idle(kc)
execute(kc=kc, code="dim(df)")
res = get_display_data(iopub)
self.assertEqual(res, '[1] 1000 10')
execute(kc=kc, code="%use sos")
wait_for_idle(kc)
def testMagicPreview(self):
with sos_kernel() as kc:
# preview variable
iopub = kc.iopub_channel
execute(kc=kc, code='''\
%preview -n a
a=1
''')
_, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
# preview csv file
execute(kc=kc,
code='''\
%preview a.csv
with open('a.csv', 'w') as csv:
csv.write("""\
a,b,c
1,2,3
4,5,6
""")
''')
res = get_display_data(iopub, 'text/html')
self.assertTrue('dataframe_container' in res,
'Expect preview {}'.format(res))
# preview txt file
execute(kc=kc,
code='''
%preview a.txt
with open('a.txt', 'w') as txt:
txt.write("""\
hello
world
""")
''')
stdout, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
self.assertTrue('world' in stdout,
'Expect preview {}'.format(stdout))
# preview zip
execute(kc=kc,
code='''
%preview -n a.zip
import zipfile
with zipfile.ZipFile('a.zip', 'w') as zfile:
zfile.write('a.csv')
''')
stdout, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
self.assertTrue('a.csv' in stdout,
'Expect preview {}'.format(stdout))
# preview tar
execute(kc=kc,
code='''
%preview -n a.tar
import tarfile
with tarfile.open('a.tar', 'w') as tar:
tar.add('a.csv')
''')
stdout, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
self.assertTrue('a.csv' in stdout,
'Expect preview {}'.format(stdout))
# preview tar.gz
execute(kc=kc,
code='''\
%preview -n a.tar.gz
import tarfile
with tarfile.open('a.tar.gz', 'w:gz') as tar:
tar.add('a.csv')
''')
stdout, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
self.assertTrue('a.csv' in stdout,
'Expect preview {}'.format(stdout))
#
# preview regular .gz
execute(kc=kc,
code='''
%preview -n a.gz
import gzip
with gzip.open('a.gz', 'w') as gz:
gz.write(b"""
Hello
world
""")
''')
stdout, stderr = get_std_output(iopub)
self.assertEqual(stderr, '')
self.assertTrue('world' in stdout,
'Expect preview {}'.format(stdout))
# preview md
execute(kc=kc,
code='''
%preview -n a.md
with open('a.md', 'w') as md:
md.write("""\
# title
* item
* item
""")
''')
res = get_display_data(iopub, 'text/html')
self.assertTrue('<li>item</li>' in res,
'Expect preview {}'.format(res))
# preview html
execute(kc=kc,
code='''
%preview -n a.html
with open('a.html', 'w') as dot:
dot.write("""\
<!DOCTYPE html>
<html>
<body>
<h1>My First Heading</h1>
<p>My first paragraph.</p>
</body>
</html>
""")
''')
res = get_display_data(iopub, 'text/html')
# preview dot, needs imagemagick, which is unavailable under windows.
if sys.platform == 'win32':
return
execute(kc=kc,
code='''
%preview -n a.dot
with open('a.dot', 'w') as dot:
dot.write("""\
graph graphname {
a -- b -- c;
b -- d;
}
""")
''')
res = get_display_data(iopub, 'image/png')
self.assertGreater(len(res), 1000, 'Expect a image {}'.format(res))