class TestServer(unittest.TestCase):
def startServer(self):
# Start the server
serverpy = os.path.join(os.path.dirname(__file__),
'start_simple_server.py')
for attempt in range(2):
self.port = 10000 + random.randrange(30000)
cflags = 0
exe = sys.executable
if sys.platform == 'win32':
if sys.version_info[:2] > (2, 6):
cflags |= subprocess.CREATE_NEW_PROCESS_GROUP
# Make sure Python.exe, not Pythonw.exe (to work around
# errors in numba in a GUI context)
exe = os.path.join(os.path.dirname(exe), 'Python.exe')
self.proc = subprocess.Popen([sys.executable,
serverpy,
self.cat.catfile,
str(self.port)],
executable=exe,
creationflags=cflags)
for i in range(30):
time.sleep(0.2)
if self.proc.poll() is not None:
break
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if s.connect_ex(('127.0.0.1',self.port)) == 0:
s.close()
return
s.close()
print("Couldn't start Blaze test server attempt %d" % attempt)
self.proc.terminate()
raise RuntimeError('Failed to start the test Blaze server')
def setUp(self):
self.cat = CatalogHarness()
# Load the test catalog for comparison with the server
blaze.catalog.load_config(self.cat.catfile)
self.startServer()
self.baseurl = 'http://localhost:%d' % self.port
def tearDown(self):
self.proc.terminate()
blaze.catalog.load_default()
self.cat.close()
def test_get_arr(self):
ra = blaze.array(RemoteDataDescriptor('%s/csv_arr' % self.baseurl))
la = blaze.catalog.get('/csv_arr')
self.assertEqual(la.dshape, ra.dshape)
self.assertEqual(dd_as_py(la._data), dd_as_py(blaze.eval(ra)._data))
def test_compute(self):
ra = blaze.array(RemoteDataDescriptor('%s/py_arr' % self.baseurl))
result = ra + 1
result = blaze.eval(result)
self.assertEqual(result.dshape, datashape.dshape('5 * int32'))
self.assertEqual(dd_as_py(result._data), [2, 3, 4, 5, 6])
class TestServer(unittest.TestCase):
def startServer(self):
# Start the server
serverpy = os.path.join(os.path.dirname(__file__),
'start_simple_server.py')
for attempt in range(2):
self.port = 10000 + random.randrange(30000)
cflags = 0
exe = sys.executable
if sys.platform == 'win32':
if sys.version_info[:2] > (2, 6):
cflags |= subprocess.CREATE_NEW_PROCESS_GROUP
# Make sure Python.exe, not Pythonw.exe (to work around
# errors in numba in a GUI context)
exe = os.path.join(os.path.dirname(exe), 'Python.exe')
self.proc = subprocess.Popen(
[sys.executable, serverpy, self.cat.catfile,
str(self.port)],
executable=exe,
creationflags=cflags)
for i in range(30):
time.sleep(0.2)
if self.proc.poll() is not None:
break
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if s.connect_ex(('127.0.0.1', self.port)) == 0:
s.close()
return
s.close()
print("Couldn't start Blaze test server attempt %d" % attempt)
self.proc.terminate()
raise RuntimeError('Failed to start the test Blaze server')
def setUp(self):
self.cat = CatalogHarness()
# Load the test catalog for comparison with the server
blaze.catalog.load_config(self.cat.catfile)
self.startServer()
self.baseurl = 'http://localhost:%d' % self.port
def tearDown(self):
self.proc.terminate()
blaze.catalog.load_default()
self.cat.close()
def test_get_arr(self):
ra = blaze.array(RemoteDataDescriptor('%s/csv_arr' % self.baseurl))
la = blaze.catalog.get('/csv_arr')
self.assertEqual(la.dshape, ra.dshape)
self.assertEqual(dd_as_py(la._data), dd_as_py(blaze.eval(ra)._data))
def test_compute(self):
ra = blaze.array(RemoteDataDescriptor('%s/py_arr' % self.baseurl))
result = ra + 1
result = blaze.eval(result)
self.assertEqual(result.dshape, datashape.dshape('5 * int32'))
self.assertEqual(dd_as_py(result._data), [2, 3, 4, 5, 6])
class TestServer(unittest.TestCase):
def startServer(self):
# Start the server
serverpy = os.path.join(os.path.dirname(__file__),
'start_simple_server.py')
for attempt in range(5):
self.port = 10000 + random.randrange(30000)
cflags = 0
if sys.platform == 'win32' and sys.version_info[:2] > (2, 6):
cflags |= subprocess.CREATE_NEW_PROCESS_GROUP
self.proc = subprocess.Popen([sys.executable,
serverpy,
self.cat.catfile,
str(self.port)],
executable=sys.executable,
creationflags=cflags)
for i in range(10):
time.sleep(0.2)
if self.proc.poll() is not None:
break
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if s.connect_ex(('127.0.0.1',self.port)) == 0:
s.close()
return
s.close()
print("Couldn't start Blaze test server attempt %d" % attempt)
self.proc.terminate()
raise RuntimeError('Failed to start the test Blaze server')
def setUp(self):
self.cat = CatalogHarness()
# Load the test catalog for comparison with the server
blaze.catalog.load_config(self.cat.catfile)
self.startServer()
self.baseurl = 'http://localhost:%d' % self.port
def tearDown(self):
self.proc.terminate()
blaze.catalog.load_default()
self.cat.close()
def test_get_arr(self):
ra = RArray('%s/csv_arr' % self.baseurl)
la = blaze.catalog.get('/csv_arr')
self.assertEqual(la.dshape, ra.dshape)
self.assertEqual(dd_as_py(la._data), dd_as_py(ra.get_data()._data))
def setUp(self):
self.cat = CatalogHarness()
blaze.catalog.load_config(self.cat.catfile)
class TestCatalog(unittest.TestCase):
def setUp(self):
self.cat = CatalogHarness()
blaze.catalog.load_config(self.cat.catfile)
def tearDown(self):
blaze.catalog.load_default()
self.cat.close()
def test_dir_traversal(self):
blaze.catalog.cd('/')
self.assertEquals(blaze.catalog.cwd(), '/')
entities = ['csv_arr', 'json_arr', 'npy_arr', 'py_arr', 'subdir']
if tables_is_here:
entities.append('hdf5_arr')
entities.sort()
self.assertEquals(blaze.catalog.ls(), entities)
arrays = ['csv_arr', 'json_arr', 'npy_arr', 'py_arr']
if tables_is_here:
arrays.append('hdf5_arr')
arrays.sort()
self.assertEquals(blaze.catalog.ls_arrs(), arrays)
self.assertEquals(blaze.catalog.ls_dirs(),
['subdir'])
blaze.catalog.cd('subdir')
self.assertEquals(blaze.catalog.cwd(), '/subdir')
self.assertEquals(blaze.catalog.ls(),
['csv_arr2'])
def test_load_csv(self):
# Confirms that a simple csv file can be loaded
blaze.catalog.cd('/')
a = blaze.catalog.get('csv_arr')
ds = datashape.dshape('5, {Letter: string; Number: int32}')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [{'Letter': 'alpha', 'Number': 0},
{'Letter': 'beta', 'Number': 1},
{'Letter': 'gamma', 'Number': 2},
{'Letter': 'delta', 'Number': 3},
{'Letter': 'epsilon', 'Number': 4}])
def test_load_json(self):
# Confirms that a simple json file can be loaded
blaze.catalog.cd('/')
a = blaze.catalog.get('json_arr')
ds = datashape.dshape('2, var, int32')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [[1, 2, 3], [1, 2]])
@skipIf(not tables_is_here, 'pytables is not installed')
def test_load_hdf5(self):
# Confirms that a simple hdf5 array in a file can be loaded
blaze.catalog.cd('/')
a = blaze.catalog.get('hdf5_arr')
ds = datashape.dshape('2, 3, int32')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [[1, 2, 3], [3, 2, 1]])
def test_load_npy(self):
# Confirms that a simple npy file can be loaded
blaze.catalog.cd('/')
a = blaze.catalog.get('npy_arr')
ds = datashape.dshape('20, {idx: int32; val: string}')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual([x['idx'] for x in dat],
list(range(20)))
self.assertEqual([x['val'] for x in dat],
['yes', 'no'] * 10)
def test_load_py(self):
# Confirms that a simple py file can generate a blaze array
blaze.catalog.cd('/')
a = blaze.catalog.get('py_arr')
ds = datashape.dshape('5, int32')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [1, 2, 3, 4, 5])
def setUp(self):
self.cat = CatalogHarness()
# Load the test catalog for comparison with the server
blaze.catalog.load_config(self.cat.catfile)
self.startServer()
self.baseurl = 'http://localhost:%d' % self.port
def setUp(self):
self.cat = CatalogHarness()
blaze.catalog.load_config(self.cat.catfile)
class TestCatalog(unittest.TestCase):
def setUp(self):
self.cat = CatalogHarness()
blaze.catalog.load_config(self.cat.catfile)
def tearDown(self):
blaze.catalog.load_default()
self.cat.close()
def test_dir_traversal(self):
blaze.catalog.cd('/')
self.assertEquals(blaze.catalog.cwd(), '/')
entities = ['csv_arr', 'json_arr', 'npy_arr', 'py_arr', 'subdir']
if tables_is_here:
entities.append('hdf5_arr')
entities.sort()
self.assertEquals(blaze.catalog.ls(), entities)
arrays = ['csv_arr', 'json_arr', 'npy_arr', 'py_arr']
if tables_is_here:
arrays.append('hdf5_arr')
arrays.sort()
self.assertEquals(blaze.catalog.ls_arrs(), arrays)
self.assertEquals(blaze.catalog.ls_dirs(),
['subdir'])
blaze.catalog.cd('subdir')
self.assertEquals(blaze.catalog.cwd(), '/subdir')
self.assertEquals(blaze.catalog.ls(),
['csv_arr2'])
def test_load_csv(self):
# Confirms that a simple csv file can be loaded
blaze.catalog.cd('/')
a = blaze.catalog.get('csv_arr')
ds = datashape.dshape('5, {Letter: string; Number: int32}')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [{'Letter': 'alpha', 'Number': 0},
{'Letter': 'beta', 'Number': 1},
{'Letter': 'gamma', 'Number': 2},
{'Letter': 'delta', 'Number': 3},
{'Letter': 'epsilon', 'Number': 4}])
def test_load_json(self):
# Confirms that a simple json file can be loaded
blaze.catalog.cd('/')
a = blaze.catalog.get('json_arr')
ds = datashape.dshape('2, var, int32')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [[1, 2, 3], [1, 2]])
@skipIf(not tables_is_here, 'pytables is not installed')
def test_load_hdf5(self):
# Confirms that a simple hdf5 array in a file can be loaded
blaze.catalog.cd('/')
a = blaze.catalog.get('hdf5_arr')
ds = datashape.dshape('2, 3, int32')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [[1, 2, 3], [3, 2, 1]])
def test_load_npy(self):
# Confirms that a simple npy file can be loaded
blaze.catalog.cd('/')
a = blaze.catalog.get('npy_arr')
ds = datashape.dshape('20, {idx: int32; val: string}')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual([x['idx'] for x in dat],
list(range(20)))
self.assertEqual([x['val'] for x in dat],
['yes', 'no'] * 10)
def test_load_py(self):
# Confirms that a simple py file can generate a blaze array
blaze.catalog.cd('/')
a = blaze.catalog.get('py_arr')
ds = datashape.dshape('5, int32')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [1, 2, 3, 4, 5])
def setUp(self):
self.cat = CatalogHarness()
# Load the test catalog for comparison with the server
blaze.catalog.load_config(self.cat.catfile)
self.startServer()
self.baseurl = 'http://localhost:%d' % self.port
class TestCatalog(unittest.TestCase):
def setUp(self):
self.cat = CatalogHarness()
blaze.catalog.load_config(self.cat.catfile)
def tearDown(self):
blaze.catalog.load_default()
self.cat.close()
def test_dir_traversal(self):
blaze.catalog.cd('/')
self.assertEquals(blaze.catalog.cwd(), '/')
entities = ['csv_arr', 'json_arr', 'npy_arr', 'py_arr', 'subdir']
if tables_is_here:
entities.append('hdf5_arr')
self.assertEquals(blaze.catalog.ls(), sorted(entities))
arrays = ['csv_arr', 'json_arr', 'npy_arr', 'py_arr']
if tables_is_here:
arrays.append('hdf5_arr')
self.assertEquals(blaze.catalog.ls_arrs(), sorted(arrays))
self.assertEquals(blaze.catalog.ls_dirs(),
['hdf5_dir', 'subdir'])
blaze.catalog.cd('subdir')
self.assertEquals(blaze.catalog.cwd(), '/subdir')
self.assertEquals(blaze.catalog.ls(),
['csv_arr2'])
def test_load_csv(self):
# Confirms that a simple csv file can be loaded
blaze.catalog.cd('/')
a = blaze.catalog.get('csv_arr')
ds = datashape.dshape('5 * {Letter: string, Number: int32}')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [{'Letter': 'alpha', 'Number': 0},
{'Letter': 'beta', 'Number': 1},
{'Letter': 'gamma', 'Number': 2},
{'Letter': 'delta', 'Number': 3},
{'Letter': 'epsilon', 'Number': 4}])
def test_load_json(self):
# Confirms that a simple json file can be loaded
blaze.catalog.cd('/')
a = blaze.catalog.get('json_arr')
ds = datashape.dshape('2 * var * int32')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [[1, 2, 3], [1, 2]])
@skipIf(not tables_is_here, 'PyTables is not installed')
def test_load_hdf5(self):
# Confirms that a simple hdf5 array in a file can be loaded
blaze.catalog.cd('/')
a = blaze.catalog.get('hdf5_arr')
ds = datashape.dshape('2 * 3 * int32')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [[1, 2, 3], [3, 2, 1]])
@skipIf(not tables_is_here, 'PyTables is not installed')
def test_hdf5_dir(self):
blaze.catalog.cd('/hdf5_dir')
self.assertEquals(blaze.catalog.cwd(), '/hdf5_dir')
self.assertEquals(blaze.catalog.ls(), sorted(['a1', 'mygroup']))
self.assertEquals(blaze.catalog.ls_dirs(), sorted(['mygroup']))
self.assertEquals(blaze.catalog.ls_arrs(), sorted(['a1']))
@skipIf(not tables_is_here, 'PyTables is not installed')
def test_hdf5_subdir(self):
blaze.catalog.cd('/hdf5_dir/mygroup')
self.assertEquals(blaze.catalog.cwd(), '/hdf5_dir/mygroup')
self.assertEquals(blaze.catalog.ls(),
sorted(['a2', 'a3', 'mygroup2']))
self.assertEquals(blaze.catalog.ls_dirs(), sorted(['mygroup2']))
self.assertEquals(blaze.catalog.ls_arrs(), sorted(['a2', 'a3']))
@skipIf(not tables_is_here, 'PyTables is not installed')
def test_hdf5_subdir_get(self):
blaze.catalog.cd('/hdf5_dir/mygroup')
a = blaze.catalog.get('a3')
ds = datashape.dshape('2 * 3 * int32')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [[1, 3, 2], [2, 1, 3]])
@skipIf(not tables_is_here, 'PyTables is not installed')
def test_hdf5_subdir_ls(self):
# Check top level
blaze.catalog.cd('/')
lall = blaze.catalog.ls_dirs()
self.assertEqual(lall, ['hdf5_dir', 'subdir'])
# Check HDF5 root level
blaze.catalog.cd('/hdf5_dir')
larrs = blaze.catalog.ls_arrs()
self.assertEqual(larrs, ['a1'])
ldirs = blaze.catalog.ls_dirs()
self.assertEqual(ldirs, ['mygroup'])
lall = blaze.catalog.ls()
self.assertEqual(lall, ['a1', 'mygroup'])
# Check HDF5 second level
blaze.catalog.cd('/hdf5_dir/mygroup')
larrs = blaze.catalog.ls_arrs()
self.assertEqual(larrs, ['a2', 'a3'])
ldirs = blaze.catalog.ls_dirs()
self.assertEqual(ldirs, ['mygroup2'])
lall = blaze.catalog.ls()
self.assertEqual(lall, ['a2', 'a3', 'mygroup2'])
def test_load_npy(self):
# Confirms that a simple npy file can be loaded
blaze.catalog.cd('/')
a = blaze.catalog.get('npy_arr')
ds = datashape.dshape('20 * {idx: int32, val: string}')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual([x['idx'] for x in dat],
list(range(20)))
self.assertEqual([x['val'] for x in dat],
['yes', 'no'] * 10)
def test_load_py(self):
# Confirms that a simple py file can generate a blaze array
blaze.catalog.cd('/')
a = blaze.catalog.get('py_arr')
ds = datashape.dshape('5 * int32')
self.assertEqual(a.dshape, ds)
dat = blaze.datadescriptor.dd_as_py(a._data)
self.assertEqual(dat, [1, 2, 3, 4, 5])
