def test_schema_append_to_file(self):
file = "{p}/test_write_catalog.py".format(p=test_file_path)
if os.path.isfile(file):
os.remove(file)
rel_to_add1 = 'public:adhoc:test1'
rel_to_add2 = 'public:adhoc:test2'
FromFileCatalog.scheme_write_to_file(
file, rel_to_add1,
"{'columnNames': ['grpID1'], 'columnTypes': ['LONG_TYPE']}")
FromFileCatalog.scheme_write_to_file(
file, rel_to_add2,
"{'columnNames': ['grpID2'], 'columnTypes': ['STRING_TYPE']}")
cut = FromFileCatalog.load_from_file(
"{p}/test_write_catalog.py".format(p=test_file_path))
self.assertEqual(cut.get_scheme(rel_to_add1).get_names(),
['grpID1'])
self.assertEqual(cut.get_scheme(rel_to_add1).get_types(),
['LONG_TYPE'])
self.assertEqual(cut.get_scheme(rel_to_add2).get_names(),
['grpID2'])
self.assertEqual(cut.get_scheme(rel_to_add2).get_types(),
['STRING_TYPE'])
def test_schema_to_file_no_append(self):
file = "{p}/test_write_catalog.py".format(p=test_file_path)
if os.path.isfile(file):
os.remove(file)
rel_to_add = 'public:adhoc:test'
FromFileCatalog.scheme_write_to_file(
"{p}/test_write_catalog.py".format(p=test_file_path), rel_to_add,
"{'columnNames': ['grpID'], 'columnTypes': ['LONG_TYPE']}")
with self.assertRaises(IOError):
FromFileCatalog.scheme_write_to_file(
"{p}/test_write_catalog.py".format(p=test_file_path),
rel_to_add,
"{'columnNames': ['grpID'], 'columnTypes': ['LONG_TYPE']}",
append=False)
currentChunksToMove = numberChunksToMove[positionCount]
chunkTimes = []
runtime = 0
for c in range(1,currentChunksToMove+1):
#clear cache
subprocess.call(['/bin/bash',"../queries/clear-tpch.sh"])
print("postgres and os cleared")
chunkRead = 'public:adhoc10GB' + str(startWorkers) + 'WorkersChunks' + str(correspondingChunks[positionCount]) + ':lineitemPart1'
load = 'scan('+ str(chunkRead) + ')'
store = 'public:adhoc10GBFromDisk'+ str(startWorkers) + 'to' + str(r) + 'part' + str(c) + ':lineitem'
current_query = 'T1 = ' + load + '; Store(T1,' + store + ');';
#schema
FromFileCatalog.scheme_write_to_file(path='schema.py',new_rel_key=chunkRead, new_rel_schema=str(schema))
catalog = FromFileCatalog.load_from_file('schema.py')
_parser = parser.Parser()
statement_list = _parser.parse(current_query);
processor = interpreter.StatementProcessor(catalog, True)
processor.evaluate(statement_list)
p = processor.get_logical_plan()
#modify p
tail = p.args[0].input
p.args[0].input = alg.Shuffle(tail, [UnnamedAttributeRef(0), UnnamedAttributeRef(1)])
p = processor.get_physical_plan()
p.input.input.input = MyriaQueryScan(sql="select * from \"" + chunkRead + "\"", scheme=Scheme(zippedSchema))
finalplan = compile_to_json('chunkQuery',p,p)
#modify json