class TestServer(unittest.TestCase): def startServer(self): # Start the server serverpy = os.path.join(os.path.dirname(__file__), 'start_simple_server.py') for attempt in range(2): self.port = 10000 + random.randrange(30000) cflags = 0 exe = sys.executable if sys.platform == 'win32': if sys.version_info[:2] > (2, 6): cflags |= subprocess.CREATE_NEW_PROCESS_GROUP # Make sure Python.exe, not Pythonw.exe (to work around # errors in numba in a GUI context) exe = os.path.join(os.path.dirname(exe), 'Python.exe') self.proc = subprocess.Popen([sys.executable, serverpy, self.cat.catfile, str(self.port)], executable=exe, creationflags=cflags) for i in range(30): time.sleep(0.2) if self.proc.poll() is not None: break s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if s.connect_ex(('127.0.0.1',self.port)) == 0: s.close() return s.close() print("Couldn't start Blaze test server attempt %d" % attempt) self.proc.terminate() raise RuntimeError('Failed to start the test Blaze server') def setUp(self): self.cat = CatalogHarness() # Load the test catalog for comparison with the server blaze.catalog.load_config(self.cat.catfile) self.startServer() self.baseurl = 'http://localhost:%d' % self.port def tearDown(self): self.proc.terminate() blaze.catalog.load_default() self.cat.close() def test_get_arr(self): ra = blaze.array(RemoteDataDescriptor('%s/csv_arr' % self.baseurl)) la = blaze.catalog.get('/csv_arr') self.assertEqual(la.dshape, ra.dshape) self.assertEqual(dd_as_py(la._data), dd_as_py(blaze.eval(ra)._data)) def test_compute(self): ra = blaze.array(RemoteDataDescriptor('%s/py_arr' % self.baseurl)) result = ra + 1 result = blaze.eval(result) self.assertEqual(result.dshape, datashape.dshape('5 * int32')) self.assertEqual(dd_as_py(result._data), [2, 3, 4, 5, 6])
class TestServer(unittest.TestCase): def startServer(self): # Start the server serverpy = os.path.join(os.path.dirname(__file__), 'start_simple_server.py') for attempt in range(2): self.port = 10000 + random.randrange(30000) cflags = 0 exe = sys.executable if sys.platform == 'win32': if sys.version_info[:2] > (2, 6): cflags |= subprocess.CREATE_NEW_PROCESS_GROUP # Make sure Python.exe, not Pythonw.exe (to work around # errors in numba in a GUI context) exe = os.path.join(os.path.dirname(exe), 'Python.exe') self.proc = subprocess.Popen( [sys.executable, serverpy, self.cat.catfile, str(self.port)], executable=exe, creationflags=cflags) for i in range(30): time.sleep(0.2) if self.proc.poll() is not None: break s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if s.connect_ex(('127.0.0.1', self.port)) == 0: s.close() return s.close() print("Couldn't start Blaze test server attempt %d" % attempt) self.proc.terminate() raise RuntimeError('Failed to start the test Blaze server') def setUp(self): self.cat = CatalogHarness() # Load the test catalog for comparison with the server blaze.catalog.load_config(self.cat.catfile) self.startServer() self.baseurl = 'http://localhost:%d' % self.port def tearDown(self): self.proc.terminate() blaze.catalog.load_default() self.cat.close() def test_get_arr(self): ra = blaze.array(RemoteDataDescriptor('%s/csv_arr' % self.baseurl)) la = blaze.catalog.get('/csv_arr') self.assertEqual(la.dshape, ra.dshape) self.assertEqual(dd_as_py(la._data), dd_as_py(blaze.eval(ra)._data)) def test_compute(self): ra = blaze.array(RemoteDataDescriptor('%s/py_arr' % self.baseurl)) result = ra + 1 result = blaze.eval(result) self.assertEqual(result.dshape, datashape.dshape('5 * int32')) self.assertEqual(dd_as_py(result._data), [2, 3, 4, 5, 6])
class TestServer(unittest.TestCase): def startServer(self): # Start the server serverpy = os.path.join(os.path.dirname(__file__), 'start_simple_server.py') for attempt in range(5): self.port = 10000 + random.randrange(30000) cflags = 0 if sys.platform == 'win32' and sys.version_info[:2] > (2, 6): cflags |= subprocess.CREATE_NEW_PROCESS_GROUP self.proc = subprocess.Popen([sys.executable, serverpy, self.cat.catfile, str(self.port)], executable=sys.executable, creationflags=cflags) for i in range(10): time.sleep(0.2) if self.proc.poll() is not None: break s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if s.connect_ex(('127.0.0.1',self.port)) == 0: s.close() return s.close() print("Couldn't start Blaze test server attempt %d" % attempt) self.proc.terminate() raise RuntimeError('Failed to start the test Blaze server') def setUp(self): self.cat = CatalogHarness() # Load the test catalog for comparison with the server blaze.catalog.load_config(self.cat.catfile) self.startServer() self.baseurl = 'http://localhost:%d' % self.port def tearDown(self): self.proc.terminate() blaze.catalog.load_default() self.cat.close() def test_get_arr(self): ra = RArray('%s/csv_arr' % self.baseurl) la = blaze.catalog.get('/csv_arr') self.assertEqual(la.dshape, ra.dshape) self.assertEqual(dd_as_py(la._data), dd_as_py(ra.get_data()._data))
def setUp(self): self.cat = CatalogHarness() blaze.catalog.load_config(self.cat.catfile)
class TestCatalog(unittest.TestCase): def setUp(self): self.cat = CatalogHarness() blaze.catalog.load_config(self.cat.catfile) def tearDown(self): blaze.catalog.load_default() self.cat.close() def test_dir_traversal(self): blaze.catalog.cd('/') self.assertEquals(blaze.catalog.cwd(), '/') entities = ['csv_arr', 'json_arr', 'npy_arr', 'py_arr', 'subdir'] if tables_is_here: entities.append('hdf5_arr') entities.sort() self.assertEquals(blaze.catalog.ls(), entities) arrays = ['csv_arr', 'json_arr', 'npy_arr', 'py_arr'] if tables_is_here: arrays.append('hdf5_arr') arrays.sort() self.assertEquals(blaze.catalog.ls_arrs(), arrays) self.assertEquals(blaze.catalog.ls_dirs(), ['subdir']) blaze.catalog.cd('subdir') self.assertEquals(blaze.catalog.cwd(), '/subdir') self.assertEquals(blaze.catalog.ls(), ['csv_arr2']) def test_load_csv(self): # Confirms that a simple csv file can be loaded blaze.catalog.cd('/') a = blaze.catalog.get('csv_arr') ds = datashape.dshape('5, {Letter: string; Number: int32}') self.assertEqual(a.dshape, ds) dat = blaze.datadescriptor.dd_as_py(a._data) self.assertEqual(dat, [{'Letter': 'alpha', 'Number': 0}, {'Letter': 'beta', 'Number': 1}, {'Letter': 'gamma', 'Number': 2}, {'Letter': 'delta', 'Number': 3}, {'Letter': 'epsilon', 'Number': 4}]) def test_load_json(self): # Confirms that a simple json file can be loaded blaze.catalog.cd('/') a = blaze.catalog.get('json_arr') ds = datashape.dshape('2, var, int32') self.assertEqual(a.dshape, ds) dat = blaze.datadescriptor.dd_as_py(a._data) self.assertEqual(dat, [[1, 2, 3], [1, 2]]) @skipIf(not tables_is_here, 'pytables is not installed') def test_load_hdf5(self): # Confirms that a simple hdf5 array in a file can be loaded blaze.catalog.cd('/') a = blaze.catalog.get('hdf5_arr') ds = datashape.dshape('2, 3, int32') self.assertEqual(a.dshape, ds) dat = blaze.datadescriptor.dd_as_py(a._data) self.assertEqual(dat, [[1, 2, 3], [3, 2, 1]]) def test_load_npy(self): # Confirms that a simple npy file can be loaded blaze.catalog.cd('/') a = blaze.catalog.get('npy_arr') ds = datashape.dshape('20, {idx: int32; val: string}') self.assertEqual(a.dshape, ds) dat = blaze.datadescriptor.dd_as_py(a._data) self.assertEqual([x['idx'] for x in dat], list(range(20))) self.assertEqual([x['val'] for x in dat], ['yes', 'no'] * 10) def test_load_py(self): # Confirms that a simple py file can generate a blaze array blaze.catalog.cd('/') a = blaze.catalog.get('py_arr') ds = datashape.dshape('5, int32') self.assertEqual(a.dshape, ds) dat = blaze.datadescriptor.dd_as_py(a._data) self.assertEqual(dat, [1, 2, 3, 4, 5])
def setUp(self): self.cat = CatalogHarness() # Load the test catalog for comparison with the server blaze.catalog.load_config(self.cat.catfile) self.startServer() self.baseurl = 'http://localhost:%d' % self.port
class TestCatalog(unittest.TestCase): def setUp(self): self.cat = CatalogHarness() blaze.catalog.load_config(self.cat.catfile) def tearDown(self): blaze.catalog.load_default() self.cat.close() def test_dir_traversal(self): blaze.catalog.cd('/') self.assertEquals(blaze.catalog.cwd(), '/') entities = ['csv_arr', 'json_arr', 'npy_arr', 'py_arr', 'subdir'] if tables_is_here: entities.append('hdf5_arr') self.assertEquals(blaze.catalog.ls(), sorted(entities)) arrays = ['csv_arr', 'json_arr', 'npy_arr', 'py_arr'] if tables_is_here: arrays.append('hdf5_arr') self.assertEquals(blaze.catalog.ls_arrs(), sorted(arrays)) self.assertEquals(blaze.catalog.ls_dirs(), ['hdf5_dir', 'subdir']) blaze.catalog.cd('subdir') self.assertEquals(blaze.catalog.cwd(), '/subdir') self.assertEquals(blaze.catalog.ls(), ['csv_arr2']) def test_load_csv(self): # Confirms that a simple csv file can be loaded blaze.catalog.cd('/') a = blaze.catalog.get('csv_arr') ds = datashape.dshape('5 * {Letter: string, Number: int32}') self.assertEqual(a.dshape, ds) dat = blaze.datadescriptor.dd_as_py(a._data) self.assertEqual(dat, [{'Letter': 'alpha', 'Number': 0}, {'Letter': 'beta', 'Number': 1}, {'Letter': 'gamma', 'Number': 2}, {'Letter': 'delta', 'Number': 3}, {'Letter': 'epsilon', 'Number': 4}]) def test_load_json(self): # Confirms that a simple json file can be loaded blaze.catalog.cd('/') a = blaze.catalog.get('json_arr') ds = datashape.dshape('2 * var * int32') self.assertEqual(a.dshape, ds) dat = blaze.datadescriptor.dd_as_py(a._data) self.assertEqual(dat, [[1, 2, 3], [1, 2]]) @skipIf(not tables_is_here, 'PyTables is not installed') def test_load_hdf5(self): # Confirms that a simple hdf5 array in a file can be loaded blaze.catalog.cd('/') a = blaze.catalog.get('hdf5_arr') ds = datashape.dshape('2 * 3 * int32') self.assertEqual(a.dshape, ds) dat = blaze.datadescriptor.dd_as_py(a._data) self.assertEqual(dat, [[1, 2, 3], [3, 2, 1]]) @skipIf(not tables_is_here, 'PyTables is not installed') def test_hdf5_dir(self): blaze.catalog.cd('/hdf5_dir') self.assertEquals(blaze.catalog.cwd(), '/hdf5_dir') self.assertEquals(blaze.catalog.ls(), sorted(['a1', 'mygroup'])) self.assertEquals(blaze.catalog.ls_dirs(), sorted(['mygroup'])) self.assertEquals(blaze.catalog.ls_arrs(), sorted(['a1'])) @skipIf(not tables_is_here, 'PyTables is not installed') def test_hdf5_subdir(self): blaze.catalog.cd('/hdf5_dir/mygroup') self.assertEquals(blaze.catalog.cwd(), '/hdf5_dir/mygroup') self.assertEquals(blaze.catalog.ls(), sorted(['a2', 'a3', 'mygroup2'])) self.assertEquals(blaze.catalog.ls_dirs(), sorted(['mygroup2'])) self.assertEquals(blaze.catalog.ls_arrs(), sorted(['a2', 'a3'])) @skipIf(not tables_is_here, 'PyTables is not installed') def test_hdf5_subdir_get(self): blaze.catalog.cd('/hdf5_dir/mygroup') a = blaze.catalog.get('a3') ds = datashape.dshape('2 * 3 * int32') self.assertEqual(a.dshape, ds) dat = blaze.datadescriptor.dd_as_py(a._data) self.assertEqual(dat, [[1, 3, 2], [2, 1, 3]]) @skipIf(not tables_is_here, 'PyTables is not installed') def test_hdf5_subdir_ls(self): # Check top level blaze.catalog.cd('/') lall = blaze.catalog.ls_dirs() self.assertEqual(lall, ['hdf5_dir', 'subdir']) # Check HDF5 root level blaze.catalog.cd('/hdf5_dir') larrs = blaze.catalog.ls_arrs() self.assertEqual(larrs, ['a1']) ldirs = blaze.catalog.ls_dirs() self.assertEqual(ldirs, ['mygroup']) lall = blaze.catalog.ls() self.assertEqual(lall, ['a1', 'mygroup']) # Check HDF5 second level blaze.catalog.cd('/hdf5_dir/mygroup') larrs = blaze.catalog.ls_arrs() self.assertEqual(larrs, ['a2', 'a3']) ldirs = blaze.catalog.ls_dirs() self.assertEqual(ldirs, ['mygroup2']) lall = blaze.catalog.ls() self.assertEqual(lall, ['a2', 'a3', 'mygroup2']) def test_load_npy(self): # Confirms that a simple npy file can be loaded blaze.catalog.cd('/') a = blaze.catalog.get('npy_arr') ds = datashape.dshape('20 * {idx: int32, val: string}') self.assertEqual(a.dshape, ds) dat = blaze.datadescriptor.dd_as_py(a._data) self.assertEqual([x['idx'] for x in dat], list(range(20))) self.assertEqual([x['val'] for x in dat], ['yes', 'no'] * 10) def test_load_py(self): # Confirms that a simple py file can generate a blaze array blaze.catalog.cd('/') a = blaze.catalog.get('py_arr') ds = datashape.dshape('5 * int32') self.assertEqual(a.dshape, ds) dat = blaze.datadescriptor.dd_as_py(a._data) self.assertEqual(dat, [1, 2, 3, 4, 5])