def testCss3(self):
""" Test for saving data to CSS """
# instantiate kvI with come initial data
initData = """\
/\t\\N
/css_meta\t\\N
/css_meta/version\t{version}
/DBS\t\\N
/DBS/{db}\t\\N
/DBS/{db}/TABLES\t\\N
/DBS/{db}/TABLES/{table}\t\\N
/DBS/{db}/TABLES/{table}/CHUNKS\t\\N
"""
workers = ['worker1', 'worker2']
database = 'TESTDB'
table = 'TABLE'
initData = initData.format(version=css.VERSION,
db=database,
table=table)
css_inst = _makeCss(initData)
mapper = ChunkMapping(workers, database, table, css_inst)
# chunks that are not in CSS should return workers from the list
worker = mapper.worker(1)
self.assertEqual(worker, 'worker1')
worker = mapper.worker(2)
self.assertEqual(worker, 'worker2')
# save stuff to CSS
mapper.save()
# save all CSS to string
data = css_inst.getKvI().dumpKV()
# build another CSS instance from saved data
css_inst = _makeCss(data)
# new mapper, use different worker set to avoid confusion
workers = ['worker1000', 'worker2000']
mapper = ChunkMapping(workers, database, table, css_inst)
# get workers for chunks from css
worker = mapper.worker(1)
self.assertEqual(worker, 'worker1')
worker = mapper.worker(2)
self.assertEqual(worker, 'worker2')
worker = mapper.worker(3)
self.assertEqual(worker, 'worker1000')
worker = mapper.worker(4)
self.assertEqual(worker, 'worker2000')
def testRepeat(self):
""" Testing repeatability without CSS backend """
workers = ['worker1', 'worker2']
database = 'TESTDB'
table = 'TABLE'
mapper = ChunkMapping(workers, database, table)
# repeat the same thing twice, must get identical output
wmap1 = [mapper.worker(w) for w in range(10)]
wmap2 = [mapper.worker(w) for w in range(10)]
self.assertEqual(wmap1, wmap2)
def testRR(self):
""" Testing round-robin without CSS backend """
workers = ['worker1', 'worker2']
database = 'TESTDB'
table = 'TABLE'
mapper = ChunkMapping(workers, database, table)
# current implementation works in round-robin mode
wmap = [mapper.worker(w) for w in range(10)]
for i, worker in enumerate(wmap):
expect = workers[i % len(workers)]
self.assertEqual(worker, expect)
def _run(self, database, table, schema, data):
"""
Do loading only, cleanup is done in _cleanup()
"""
# see if database is already defined in CSS and get its partitioning info
if self.css is not None:
self._checkCss(database, table)
# make chunk mapper
self.chunkMap = ChunkMapping(list(self.workerWmgrMap.keys()), database,
table, self.css)
# make chunks directory or check that there are usable data there already
self._makeOrCheckChunksDir(data)
# uncompress data files that are compressed, this is only needed if
# table is not partitioned or if we are not reusing existing chunks
files = data
if not (self.partitioned and self.skipPart and not self.oneTable):
files = self._gunzip(data)
# run partitioner if necessary
if files and self.callPartitioner:
self._runPartitioner(files)
# drop existing tables
if self.deleteTables:
self._deleteTable(database, table)
# create table
self._createTable(database, table, schema)
# load data
self._loadData(database, table, files)
# create special dummy chunk
self._createDummyChunk(database, table)
# create index on czar size
self._makeIndex(database, table)
# update CSS with info for this table
if self.css is not None:
self._updateCss(database, table)
# optionally make emptyChunks file
self._makeEmptyChunks()
def testCss1(self):
""" Test for reading data from CSS """
# instantiate kvI with come initial data
initData = """\
/\t\\N
/css_meta\t\\N
/css_meta/version\t{version}
/DBS\t\\N
/DBS/{db}\t\\N
/DBS/{db}/TABLES\t\\N
/DBS/{db}/TABLES/{table}\t\\N
/DBS/{db}/TABLES/{table}/CHUNKS\t\\N
/DBS/{db}/TABLES/{table}/CHUNKS/333\t\\N
/DBS/{db}/TABLES/{table}/CHUNKS/333/REPLICAS\t\\N
/DBS/{db}/TABLES/{table}/CHUNKS/333/REPLICAS/1\t\\N
/DBS/{db}/TABLES/{table}/CHUNKS/333/REPLICAS/1/.packed.json\t{{"nodeName": "worker333"}}
/DBS/{db}/TABLES/{table}/CHUNKS/765\t\\N
/DBS/{db}/TABLES/{table}/CHUNKS/765/REPLICAS\t\\N
/DBS/{db}/TABLES/{table}/CHUNKS/765/REPLICAS/1\t\\N
/DBS/{db}/TABLES/{table}/CHUNKS/765/REPLICAS/1/.packed.json\t{{"nodeName": "worker765"}}
"""
workers = ['worker1', 'worker2']
database = 'TESTDB'
table = 'TABLE'
initData = initData.format(version=css.VERSION,
db=database,
table=table)
css_inst = _makeCss(initData)
mapper = ChunkMapping(workers, database, table, css_inst)
# check that pre-defined chunks work
worker = mapper.worker(333)
self.assertEqual(worker, 'worker333')
worker = mapper.worker(765)
self.assertEqual(worker, 'worker765')
# chunks that are not in CSS should return workers from the list
worker = mapper.worker(1)
self.assertEqual(worker, 'worker1')
worker = mapper.worker(2)
self.assertEqual(worker, 'worker2')