def testShell(self): with sos_kernel() as kc: iopub = kc.iopub_channel execute(kc=kc, code="!echo ha ha") stdout, stderr = get_std_output(iopub) self.assertTrue('ha ha' in stdout, "GOT ERROR {}".format(stderr)) self.assertEqual(stderr, '')
def testGetPythonDataFrameFromMatlab(self): # Python -> Matlab/Matlab with sos_kernel() as kc: iopub = kc.iopub_channel # create a data frame execute(kc=kc, code=''' import pandas as pd import numpy as np import scipy.io as sio arr = np.random.randn(1000) arr[::10] = np.nan df = pd.DataFrame({'column_{0}'.format(i): arr for i in range(10)}) ''') clear_channels(iopub) execute(kc=kc, code="%use MATLAB") wait_for_idle(kc) #_, stderr = assemble_output(iopub) #self.assertEqual(stderr, '') execute(kc=kc, code="%get df") wait_for_idle(kc) execute(kc=kc, code="display(size(df))") stdout, _ = assemble_output(iopub) self.assertEqual(stdout.strip().split(), ['1000', '10']) execute(kc=kc, code="%use sos") wait_for_idle(kc)
def testReverseSharedVariable(self): '''Test shared variables defined in auxiliary steps''' if file_target('a.txt').exists(): file_target('a.txt').unlink() script = r''' %run B [A: shared='b', provides='a.txt'] b = 1 run: touch a.txt [B_1] depends: 'a.txt' [B_2] print(b) ''' with sos_kernel() as kc: iopub = kc.iopub_channel execute(kc=kc, code=script) wait_for_idle(kc) self.assertTrue(os.path.isfile('a.txt')) execute(kc=kc, code="b") res = get_result(iopub) self.assertEqual(res, 1)
def testPythonToJavaDataframe(self): with sos_kernel() as kc: iopub = kc.iopub_channel execute(kc=kc, code=''' import numpy as np import pandas as pd dataframe = pd.DataFrame(np.random.randn(1000,4), columns=list('ABCD')) ''') wait_for_idle(kc) execute(kc=kc, code='%use Java') wait_for_idle(kc) execute(kc=kc, code='%get dataframe') wait_for_idle(kc) execute( kc=kc, code= 'System.out.println(dataframe.rowCount() * dataframe.columnCount());' ) stdout, _ = assemble_output(iopub) self.assertEqual(stdout.strip(), '4000') execute(kc=kc, code="%use sos") wait_for_idle(kc)
def testKernel(self): with sos_kernel() as kc: execute(kc=kc, code='a = 1') stdout, stderr = assemble_output(kc.get_iopub_msg) self.assertEqual(stdout.strip(), '', f'Stdout is not empty, "{stdout}" received') self.assertEqual(stderr.strip(), '', f'Stderr is not empty, "{stderr}" received')
def testMagicDebug(self): with sos_kernel() as kc: # preview variable execute(kc=kc, code=''' %debug on %debug off ''') wait_for_idle(kc)
def testMagicSessioninfo(self): with sos_kernel() as kc: # preview variable execute(kc=kc, code=''' %use Python3 %use SoS %sessioninfo ''') wait_for_idle(kc)
def testCD(self): with sos_kernel() as kc: iopub = kc.iopub_channel msg_id, content = execute(kc=kc, code="%cd ..") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="print(os.getcwd())") stdout, stderr = assemble_output(iopub) self.assertTrue(stdout.strip().endswith('jupyter')) self.assertEqual(stderr, '') msg_id, content = execute(kc=kc, code="%cd test")
def testMagicRevisions(self): with sos_kernel() as kc: iopub = kc.iopub_channel # preview variable execute(kc=kc, code=''' %revisions ''') _, stderr = get_std_output(iopub) self.assertEqual(stderr, '', f"Get error {stderr} for magic revisions")
def testGetPythonDataFromMatlab(self): with sos_kernel() as kc: iopub = kc.iopub_channel execute(kc=kc, code=''' null_var = None num_var = 123 import numpy import scipy.io as sio num_arr_var = numpy.array([1, 2, 3]) logic_var = True logic_arr_var = [True, False, True] float_var = [1.2, 2.1] char_var = '1"23' char_arr_var = ['1', '2', '3'] list_var = [1, 2, '3'] dict_var = dict(a=1, b=2, c='3') set_var = {1, 2, '3'} mat_var = numpy.matrix([[1,2],[3,4]]) recursive_var = {'a': {'b': 123}, 'c': True} ''') wait_for_idle(kc) execute(kc=kc, code=''' %use MATLAB %get null_var num_var num_arr_var logic_var logic_arr_var char_var char_arr_var mat_var set_var list_var dict_var recursive_var float_var %dict -r %put null_var num_var num_arr_var logic_var logic_arr_var char_var char_arr_var mat_var set_var list_var dict_var recursive_var float_var %use sos %dict null_var num_var num_arr_var logic_var logic_arr_var char_var char_arr_var mat_var set_var list_var dict_var recursive_var float_var ''') res = get_result(iopub) self.assertEqual(res['null_var'], None) self.assertEqual(res['num_var'], 123) self.assertEqual(list(res['num_arr_var']), [1, 2, 3], "Got {}".format(res['num_arr_var'])) self.assertEqual(res['logic_var'], True) self.assertEqual(res['logic_arr_var'], [True, False, True]) self.assertEqual(res['char_var'], '1"23') self.assertEqual(list(res['float_var']), [1.2, 2.1]) self.assertEqual(list(res['list_var']), [1, 2, '3'], 'Got {}'.format(res['list_var'])) self.assertEqual(res['dict_var'], {'a': 1, 'b': 2, 'c': '3'}) self.assertTrue( len(res['set_var']) == 3 and '3' in res['set_var'] and 1 in res['set_var'] and 2 in res['set_var']) self.assertEqual(res['mat_var'].shape, (2, 2)) self.assertEqual(res['recursive_var'], { 'a': { 'b': 123 }, 'c': True })
def testMagicSandbox(self): with sos_kernel() as kc: # preview variable execute(kc=kc, code=''' %sandbox with open('test_blah.txt', 'w') as tb: tb.write('a') ''') wait_for_idle(kc) self.assertFalse(os.path.exists('test_blah.txt'))
def testWith(self): '''Test magic with''' with sos_kernel() as kc: iopub = kc.iopub_channel # create a data frame msg_id, content = execute(kc=kc, code='var = [1, 2, 3, 4]') wait_for_idle(kc) msg_id, content = execute(kc=kc, code='%with R -i var -o m\nm=mean(var)') wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%dict m") res = get_result(iopub) self.assertEqual(res['m'], 2.5)
def testSetSigil(self): '''Test set_options of sigil''' with sos_kernel() as kc: iopub = kc.iopub_channel # create a data frame msg_id, content = execute(kc=kc, code='%set_options sigil=None') wait_for_idle(kc) msg_id, content = execute(kc=kc, code='a="${}".format(100)') wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%dict a") res = get_result(iopub) self.assertEqual(res['a'], "$100")
def testMagicSessionInfo(self): with sos_kernel() as kc: iopub = kc.iopub_channel # preview variable execute(kc=kc, code=''' %use Python3 %use SoS %sessioninfo ''') _, stderr = get_std_output(iopub) self.assertEqual(stderr, '', f"Get error {stderr} for magic sessioninfo")
def testSubKernel(self): with sos_kernel() as kc: iopub = kc.iopub_channel msg_id, content = execute(kc=kc, code="%use R") stdout, stderr = assemble_output(iopub) self.assertEqual(stderr, '') msg_id, content = execute(kc=kc, code="a <- 1024") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="a") res = get_display_data(iopub) self.assertEqual(res, '[1] 1024') msg_id, content = execute(kc=kc, code="%use sos") wait_for_idle(kc)
def testMagicMatplotlib(self): with sos_kernel() as kc: # create a data frame execute(kc=kc, code=''' %matplotlib inline In [59]: import matplotlib.pyplot as plt import numpy as np x = np.linspace(0, 10) plt.plot(x, np.sin(x), '--', linewidth=2) plt.show()''') wait_for_idle(kc)
def testMagicSave(self): with sos_kernel() as kc: if os.path.isfile('test.txt'): os.remove('test.txt') execute(kc=kc, code='''\ %preview ~/test.txt %save ~/test.txt a=1 ''') wait_for_idle(kc) with open(os.path.join(os.path.expanduser('~'), 'test.txt')) as tt: self.assertEqual(tt.read(), 'a=1\n')
def testKernel(self): with sos_kernel() as kc: execute(kc=kc, code='a = 1') stdout, stderr = assemble_output(kc.iopub_channel) self.assertEqual(stdout.strip(), '', f'Stdout is not empty, "{stdout}" received') self.assertEqual(stderr.strip(), '', f'Stderr is not empty, "{stderr}" received') execute(kc=kc, code='%use Python3\n%get a\nprint(a)') stdout, stderr = assemble_output(kc.iopub_channel) self.assertEqual(stderr.strip(), '', f'Stderr is not empty, "{stderr}" received') self.assertEqual(stdout.strip(), '1', f'Stdout should be 1, "{stdout}" received')
def testMagicRender(self): with sos_kernel() as kc: # preview variable execute(kc=kc, code=''' %render """ # header * item1 * item2 """ ''') wait_for_idle(kc)
def testMagicSet(self): # test preview of remote file with sos_kernel() as kc: iopub = kc.iopub_channel # preview variable execute(kc=kc, code=''' %set %set -v2 %set %set -v1 ''') stdout, stderr = get_std_output(iopub) self.assertEqual(stderr, '', 'Got {}'.format(stderr)) self.assertTrue('set' in stdout, 'Got {}'.format(stdout))
def testMagicRemotePreview(self): # test preview of remote file with sos_kernel() as kc: iopub = kc.iopub_channel # preview variable execute(kc=kc, code=''' %preview -n abc.txt -c ~/docker.yml -r docker %run -r docker -c ~/docker.yml run: echo abc > abc.txt ''') stdout, _ = get_std_output(iopub) #self.assertEqual(stderr, '', 'Got error {}'.format(stderr)) self.assertTrue('abc' in stdout, 'Got stdout "{}"'.format(stdout))
def testHelp(self): '''Test help messages''' with sos_kernel() as kc: # create a data frame execute(kc=kc, code='\n'.join( '%{} -h'.format(magic) for magic in ('cd', 'debug', 'dict', 'get', 'matplotlib', 'paste', 'preview', 'put', 'render', 'rerun', 'run', 'save', 'sandbox', 'set', 'sessioninfo', 'sosrun', 'sossave', 'shutdown', 'taskinfo', 'tasks', 'toc', 'use', 'with', 'pull', 'push'))) wait_for_idle(kc)
def testDAG(self): with sos_kernel() as kc: iopub = kc.iopub_channel execute(kc=kc, code='''\ %run [a] b=10 [default] sos_run('a') ''') wait_for_idle(kc) execute(kc=kc, code="b") res = get_result(iopub) self.assertEqual(res, 10)
def testInterpolation(self): with sos_kernel() as kc: iopub = kc.iopub_channel msg_id, content = execute(kc=kc, code='print("a=${100+11}")') stdout, stderr = assemble_output(iopub) self.assertTrue(stdout.endswith('a=111\n')) self.assertEqual(stderr, '')
def testShell(self): with sos_kernel() as kc: iopub = kc.iopub_channel msg_id, content = execute(kc=kc, code="!ls test_kernel.py") stdout, stderr = assemble_output(iopub) self.assertEqual(stdout, 'test_kernel.py\n') self.assertEqual(stderr, '')
def testSoSSave(self): # the test would not work because ipython file does not exist if os.path.isfile('sossave.html'): os.remove('sossave.html') with sos_kernel() as kc: iopub = kc.iopub_channel # preview variable execute(kc=kc, code=''' %sossave sossave.html --force [10] print('kkk') ''') _, stderr = get_std_output(iopub) self.assertTrue('.ipynb does not exist' in stderr, f"Get error {stderr}")
def testMagicDict(self): '''Test %dict magic''' with sos_kernel() as kc: iopub = kc.iopub_channel msg_id, content = execute(kc=kc, code="a=12345") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%dict a") self.assertEqual(get_result(iopub)['a'], 12345) msg_id, content = execute(kc=kc, code="%dict --keys") self.assertTrue('a' in get_result(iopub)) msg_id, content = execute(kc=kc, code="%dict --reset") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%dict --keys --all") res = get_result(iopub) self.assertTrue('a' not in res) for key in ('run', 'sh', 'tcsh', 'expand_pattern'): self.assertTrue(key in res)
def testPutRDataFrameToPython(self): with sos_kernel() as kc: iopub = kc.iopub_channel # create a data frame msg_id, content = execute(kc=kc, code='%use R') wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%put mtcars") stdout, stderr = assemble_output(iopub) # the message can contain "Loading required package feathre" #self.assertEqual(stderr, '') msg_id, content = execute(kc=kc, code="%use sos") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="mtcars.shape") res = get_result(iopub) self.assertEqual(res, (32, 11)) msg_id, content = execute(kc=kc, code="%use sos") wait_for_idle(kc)
def testPutRDataToPython(self): with sos_kernel() as kc: iopub = kc.iopub_channel # create a data frame execute(kc=kc, code="""\ %use R null_var = NULL num_var = 123 num_arr_var = c(1, 2, 3) logic_var = TRUE logic_arr_var = c(TRUE, FALSE, TRUE) char_var = '1\"23' char_arr_var = c(1, 2, '3') list_var = list(1, 2, '3') named_list_var = list(a=1, b=2, c='3') mat_var = matrix(c(1,2,3,4), nrow=2) recursive_var = list(a=1, b=list(c=3, d='whatever')) comp_var = 1+2i seri_var = setNames(c(1,2,3,3,3,3),c(0:5)) """) wait_for_idle(kc) execute(kc=kc, code='''\ %put null_var num_var num_arr_var logic_var logic_arr_var char_var char_arr_var mat_var list_var named_list_var recursive_var comp_var seri_var %use sos seri_var = list(seri_var) ''') wait_for_idle(kc) execute( kc=kc, code= "%dict null_var num_var num_arr_var logic_var logic_arr_var char_var char_arr_var mat_var list_var named_list_var recursive_var comp_var seri_var" ) res = get_result(iopub) self.assertEqual(res['null_var'], None) self.assertEqual(res['num_var'], 123) self.assertEqual(res['num_arr_var'], [1, 2, 3]) self.assertEqual(res['logic_var'], True) self.assertEqual(res['logic_arr_var'], [True, False, True]) self.assertEqual(res['char_var'], '1"23') self.assertEqual(res['char_arr_var'], ['1', '2', '3']) self.assertEqual(res['list_var'], [1, 2, '3']) self.assertEqual(res['named_list_var'], {'a': 1, 'b': 2, 'c': '3'}) self.assertEqual(res['mat_var'].shape, (2, 2)) self.assertEqual(res['recursive_var'], { 'a': 1, 'b': { 'c': 3, 'd': 'whatever' } }) self.assertEqual(res['comp_var'], 1 + 2j) self.assertEqual(res['seri_var'], [1, 2, 3, 3, 3, 3]) execute(kc=kc, code="%use sos") wait_for_idle(kc)
def testMagicDict(self): '''Test %dict magic''' with sos_kernel() as kc: iopub = kc.iopub_channel execute(kc=kc, code="a=12345") wait_for_idle(kc) execute(kc=kc, code="%dict a") self.assertEqual(get_result(iopub)['a'], 12345) execute(kc=kc, code="%dict --keys") self.assertTrue('a' in get_result(iopub)) execute(kc=kc, code="%dict --reset") wait_for_idle(kc) execute(kc=kc, code="%dict --keys --all") res = get_result(iopub) self.assertTrue('a' not in res) for key in ('run', 'expand_pattern'): self.assertTrue(key in res)
def testPullPush(self): '''Test set_options of sigil''' import random fname = os.path.expanduser("~/push_pull_{}.txt".format(random.randint(1, 100000))) with open(fname, 'w') as pp: pp.write('something') with sos_kernel() as kc: # create a data frame execute(kc=kc, code='%push {} --to docker -c ~/docker.yml'.format(fname)) wait_for_idle(kc) os.remove(fname) self.assertFalse(os.path.isfile(fname)) # execute(kc=kc, code='%pull {} --from docker -c ~/docker.yml'.format(fname)) _, stderr = get_std_output(kc.iopub_channel) self.assertEqual(stderr, '', 'Expect no error, get {}'.format(stderr)) self.assertTrue(os.path.isfile(fname))
def testGetPythonDataFrameFromR(self): with sos_kernel() as kc: iopub = kc.iopub_channel # create a data frame msg_id, content = execute(kc=kc, code=''' import pandas as pd import numpy as np arr = np.random.randn(1000) arr[::10] = np.nan df = pd.DataFrame({'column_{0}'.format(i): arr for i in range(10)}) ''') clear_channels(iopub) msg_id, content = execute(kc=kc, code="%use R") stdout, stderr = assemble_output(iopub) self.assertEqual(stderr, '') msg_id, content = execute(kc=kc, code="%get df") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="dim(df)") res = get_display_data(iopub) self.assertEqual(res, '[1] 1000 10') msg_id, content = execute(kc=kc, code="%use sos") wait_for_idle(kc)
def testMagicGet(self): with sos_kernel() as kc: iopub = kc.iopub_channel msg_id, content = execute(kc=kc, code="a = 1025") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="_b_a = 22") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%use R") stdout, stderr = assemble_output(iopub) self.assertEqual(stderr, '') msg_id, content = execute(kc=kc, code="%get a") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="a") res = get_display_data(iopub) self.assertEqual(res, '[1] 1025') # msg_id, content = execute(kc=kc, code="%get _b_a") wait_for_idle(kc) msg_id, content = execute(kc=kc, code=".b_a") res = get_display_data(iopub) self.assertEqual(res, '[1] 22') msg_id, content = execute(kc=kc, code="%use sos") wait_for_idle(kc)
def testGetPythonDataFromR(self): with sos_kernel() as kc: iopub = kc.iopub_channel msg_id, content = execute(kc=kc, code="null_var = None") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="num_var = 123") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="import numpy\nnum_arr_var = numpy.array([1, 2, 3])") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="logic_var = True") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="logic_arr_var = [True, False, True]") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="char_var = '123'") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="char_arr_var = ['1', '2', '3']") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="list_var = [1, 2, '3']") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="dict_var = dict(a=1, b=2, c='3')") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="set_var = {1, 2, '3'}") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="mat_var = numpy.matrix([[1,2],[3,4]])") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%use R") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%get null_var num_var num_arr_var logic_var logic_arr_var char_var char_arr_var mat_var set_var list_var dict_var") wait_for_idle(kc) # need to test passed values # but let us cheat by passing data back msg_id, content = execute(kc=kc, code="%dict -r") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%put null_var num_var num_arr_var logic_var logic_arr_var char_var char_arr_var mat_var set_var list_var dict_var") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%use sos") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%dict null_var num_var num_arr_var logic_var logic_arr_var char_var char_arr_var mat_var set_var list_var dict_var") res = get_result(iopub) self.assertEqual(res['null_var'], None) self.assertEqual(res['num_var'], 123) self.assertEqual(res['num_arr_var'], [1,2,3]) self.assertEqual(res['logic_var'], True) self.assertEqual(res['logic_arr_var'], [True, False, True]) self.assertEqual(res['char_var'], '123') self.assertEqual(res['char_arr_var'], ['1', '2', '3']) self.assertEqual(res['list_var'], [1,2,'3']) self.assertEqual(res['dict_var'], {'a': 1, 'b': 2, 'c': '3'}) self.assertEqual(res['mat_var'].shape, (2,2))
def testPutRDataToPython(self): with sos_kernel() as kc: iopub = kc.iopub_channel # create a data frame msg_id, content = execute(kc=kc, code='%use R') wait_for_idle(kc) msg_id, content = execute(kc=kc, code="null_var = NULL") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="num_var = 123") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="num_arr_var = c(1, 2, 3)") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="logic_var = TRUE") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="logic_arr_var = c(TRUE, FALSE, TRUE)") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="char_var = '123'") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="char_arr_var = c(1, 2, '3')") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="list_var = list(1, 2, '3')") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="named_list_var = list(a=1, b=2, c='3')") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="mat_var = matrix(c(1,2,3,4), nrow=2)") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%put null_var num_var num_arr_var logic_var logic_arr_var char_var char_arr_var mat_var list_var named_list_var") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%dict null_var num_var num_arr_var logic_var logic_arr_var char_var char_arr_var mat_var list_var named_list_var") res = get_result(iopub) self.assertEqual(res['null_var'], None) self.assertEqual(res['num_var'], 123) self.assertEqual(res['num_arr_var'], [1,2,3]) self.assertEqual(res['logic_var'], True) self.assertEqual(res['logic_arr_var'], [True, False, True]) self.assertEqual(res['char_var'], '123') self.assertEqual(res['char_arr_var'], ['1', '2', '3']) self.assertEqual(res['list_var'], [1,2,'3']) self.assertEqual(res['named_list_var'], {'a': 1, 'b': 2, 'c': '3'}) self.assertEqual(res['mat_var'].shape, (2,2)) msg_id, content = execute(kc=kc, code="%use sos") wait_for_idle(kc)
def testMagicPut(self): with sos_kernel() as kc: iopub = kc.iopub_channel msg_id, content = execute(kc=kc, code="%use R") stdout, stderr = assemble_output(iopub) self.assertEqual(stderr, '') msg_id, content = execute(kc=kc, code="a <- 1024") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%put a") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%put __k_k") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%use sos") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="a") res = get_result(iopub) self.assertEqual(res, 1024) # strange name msg_id, content = execute(kc=kc, code="%use R") wait_for_idle(kc) msg_id, content = execute(kc=kc, code=".a.b <- 22") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%put .a.b") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="%use sos") wait_for_idle(kc) msg_id, content = execute(kc=kc, code="_a_b") res = get_result(iopub) self.assertEqual(res, 22)