def test_exec2_xorsum(self):
h2o.beta_features = True
SYNDATASETS_DIR = h2o.make_syn_dir()
tryList = [
(10000, 1, 'r1', 0, 10, None),
]
ullResultList = []
for (rowCount, colCount, hex_key, expectedMin, expectedMax, expected) in tryList:
SEEDPERFILE = random.randint(0, sys.maxint)
# dynamic range of the data may be useful for estimating error
maxDelta = expectedMax - expectedMin
csvFilename = 'syn_real_' + str(rowCount) + 'x' + str(colCount) + '.csv'
csvPathname = SYNDATASETS_DIR + '/' + csvFilename
csvPathnameFull = h2i.find_folder_and_filename(None, csvPathname, returnFullPath=True)
print "Creating random", csvPathname
(expectedUll, expectedFpSum) = write_syn_dataset(csvPathname,
rowCount, colCount, expectedMin, expectedMax, SEEDPERFILE)
parseResult = h2i.import_parse(path=csvPathname, schema='local', hex_key=hex_key,
timeoutSecs=3000, retryDelaySecs=2)
inspect = h2o_cmd.runInspect(key=hex_key)
print "numRows:", inspect['numRows']
print "numCols:", inspect['numCols']
inspect = h2o_cmd.runInspect(key=hex_key, offset=-1)
print "inspect offset = -1:", h2o.dump_json(inspect)
# looking at the 8 bytes of bits for the h2o doubles
# xorsum will zero out the sign and exponent
for execExpr in exprList:
start = time.time()
(execResult, fpResult) = h2e.exec_expr(h2o.nodes[0], execExpr,
resultKey=None, timeoutSecs=300)
print 'exec took', time.time() - start, 'seconds'
print "execResult:", h2o.dump_json(execResult)
print ""
print "%30s" % "fpResult:", "%.15f" % fpResult
ullResult = h2o_util.doubleToUnsignedLongLong(fpResult)
print "%30s" % "bitResult (0.16x):", "0x%0.16x" % ullResult
print "%30s" % "expectedUll (0.16x):", "0x%0.16x" % expectedUll
# print "%30s" % "hex(bitResult):", hex(ullResult)
ullResultList.append((ullResult, fpResult))
h2o.check_sandbox_for_errors()
print "first result was from a sum. others are xorsum"
print "ullResultList:"
for ullResult, fpResult in ullResultList:
print "%30s" % "ullResult (0.16x):", "0x%0.16x %s" % (ullResult, fpResult)
expectedUllAsDouble = h2o_util.unsignedLongLongToDouble(expectedUll)
print "%30s" % "expectedUll (0.16x):", "0x%0.16x %s" % (expectedUll, expectedUllAsDouble)
expectedFpSumAsLongLong = h2o_util.doubleToUnsignedLongLong(expectedFpSum)
print "%30s" % "expectedFpSum (0.16x):", "0x%0.16x %s" % (expectedFpSumAsLongLong, expectedFpSum)
def test_exec2_xorsum(self):
SYNDATASETS_DIR = h2o.make_syn_dir()
tryList = [
(ROWS, 1, 'r1', 0, 10, None),
]
for trial in range(10):
ullResultList = []
for (rowCount, colCount, hex_key, expectedMin, expectedMax, expected) in tryList:
SEEDPERFILE = random.randint(0, sys.maxint)
# dynamic range of the data may be useful for estimating error
maxDelta = expectedMax - expectedMin
csvFilename = 'syn_real_' + str(rowCount) + 'x' + str(colCount) + '.csv'
csvPathname = SYNDATASETS_DIR + '/' + csvFilename
csvPathnameFull = h2i.find_folder_and_filename(None, csvPathname, returnFullPath=True)
print "Creating random", csvPathname
(expectedUllSum, expectedFpSum) = write_syn_dataset(csvPathname,
rowCount, colCount, expectedMin, expectedMax, SEEDPERFILE)
expectedUllSumAsDouble = h2o_util.unsignedLongLongToDouble(expectedUllSum)
expectedFpSumAsLongLong = h2o_util.doubleToUnsignedLongLong(expectedFpSum)
parseResult = h2i.import_parse(path=csvPathname, schema='put', hex_key=hex_key,
timeoutSecs=3000, retryDelaySecs=2)
numRows, numCols, parse_key = h2o_cmd.infoFromParse(parseResult)
assert parse_key == hex_key
assert numCols == colCount
assert numRows == rowCount
inspect = h2o_cmd.runInspect(key=hex_key)
missingList, labelList, numRows, numCols = h2o_cmd.infoFromInspect(inspect)
assert len(missingList) == 0
# looking at the 8 bytes of bits for the h2o doubles
# xorsum will zero out the sign and exponent
for execExpr in exprList:
for r in range(10):
start = time.time()
execResult = h2o_cmd.runExec(ast=execExpr, timeoutSecs=30)
fpResult = execResult['scalar']
# (execResult, fpResult) = h2e.exec_expr(h2o.nodes[0], execExpr, resultKey='h', timeoutSecs=300)
print r, 'exec took', time.time() - start, 'seconds'
print r, "execResult:", h2o.dump_json(execResult)
h2o_cmd.runStoreView()
ullResult = h2o_util.doubleToUnsignedLongLong(fpResult)
ullResultList.append((ullResult, fpResult))
print "%30s" % "ullResult (0.16x):", "0x%0.16x %s" % (ullResult, fpResult)
print "%30s" % "expectedUllSum (0.16x):", "0x%0.16x %s" % (expectedUllSum, expectedUllSumAsDouble)
print "%30s" % "expectedFpSum (0.16x):", "0x%0.16x %s" % (expectedFpSumAsLongLong, expectedFpSum)
# allow diff of the lsb..either way
# if ullResult!=expectedUllSum and abs((ullResult-expectedUllSum)>3):
if ullResult!=expectedUllSum:
raise Exception("h2o didn't get the same xorsum as python. 0x%0.16x 0x%0.16x" % (ullResult, expectedUllSum))
print "h2o didn't get the same xorsum as python. 0x%0.16x 0x%0.16x" % (ullResult, expectedUllSum)
h2o.check_sandbox_for_errors()
print "first result was from a sum. others are xorsum"
print "ullResultList:"
for ullResult, fpResult in ullResultList:
print "%30s" % "ullResult (0.16x):", "0x%0.16x %s" % (ullResult, fpResult)
print "%30s" % "expectedUllSum (0.16x):", "0x%0.16x %s" % (expectedUllSum, expectedUllSumAsDouble)
print "%30s" % "expectedFpSum (0.16x):", "0x%0.16x %s" % (expectedFpSumAsLongLong, expectedFpSum)
def test_exec2_xorsum2(self):
h2o.beta_features = True
SYNDATASETS_DIR = h2o.make_syn_dir()
tryList = [
(ROWS, 1, 'r1', 0, 10, None),
]
for trial in range(20):
ullResultList = []
NUM_FORMAT_CASES = h2o_util.fp_format()
for (rowCount, colCount, hex_key, expectedMin, expectedMax, expected) in tryList:
SEEDPERFILE = random.randint(0, sys.maxint)
# dynamic range of the data may be useful for estimating error
maxDelta = expectedMax - expectedMin
csvFilename = 'syn_real_' + str(rowCount) + 'x' + str(colCount) + '.csv'
csvPathname = SYNDATASETS_DIR + '/' + csvFilename
csvPathnameFull = h2i.find_folder_and_filename(None, csvPathname, returnFullPath=True)
print "Creating random", csvPathname
sel = random.randint(0, NUM_FORMAT_CASES-1)
(expectedUllSum, expectedFpSum) = write_syn_dataset(csvPathname,
rowCount, colCount, expectedMin, expectedMax, SEEDPERFILE, sel)
expectedUllSumAsDouble = h2o_util.unsignedLongLongToDouble(expectedUllSum)
expectedFpSumAsLongLong = h2o_util.doubleToUnsignedLongLong(expectedFpSum)
parseResult = h2i.import_parse(path=csvPathname, schema='local', hex_key=hex_key,
timeoutSecs=3000, retryDelaySecs=2)
inspect = h2o_cmd.runInspect(key=hex_key)
print "numRows:", inspect['numRows']
print "numCols:", inspect['numCols']
inspect = h2o_cmd.runInspect(key=hex_key, offset=-1)
print "inspect offset = -1:", h2o.dump_json(inspect)
# looking at the 8 bytes of bits for the h2o doubles
# xorsum will zero out the sign and exponent
for execExpr in exprList:
start = time.time()
(execResult, fpResult) = h2e.exec_expr(h2o.nodes[0], execExpr,
resultKey=None, timeoutSecs=300)
print 'exec took', time.time() - start, 'seconds'
print "execResult:", h2o.dump_json(execResult)
ullResult = h2o_util.doubleToUnsignedLongLong(fpResult)
ullResultList.append((ullResult, fpResult))
print "%30s" % "ullResult (0.16x):", "0x%0.16x %s" % (ullResult, fpResult)
print "%30s" % "expectedUllSum (0.16x):", "0x%0.16x %s" % (expectedUllSum, expectedUllSumAsDouble)
print "%30s" % "expectedFpSum (0.16x):", "0x%0.16x %s" % (expectedFpSumAsLongLong, expectedFpSum)
# allow diff of the lsb..either way. needed when integers are parsed
# okay for a couple of lsbs to be wrong, due to conversion from stringk
# ullResult (0.16x): 0x02c1a21f923cee96 2.15698793923e-295
# expectedUllSum (0.16x): 0x02c1a21f923cee97 2.15698793923e-295
# expectedFpSum (0.16x): 0x42f054af32b3c408 2.87294442126e+14
# ullResult and expectedUllSum are Q ints, (64-bit) so can subtract them.
# I guess we don't even care about sign, since we zero the first 4 bits (xorsum) to avoid nan/inf issues
ALLOWED_BIT_ERR = 0x1f # seeing this amount of error!
if ullResult!=expectedUllSum and (abs(ullResult-expectedUllSum)>ALLOWED_BIT_ERR):
raise Exception("h2o didn't get the same xorsum as python. 0x%0.16x 0x%0.16x" % (ullResult, expectedUllSum))
print "h2o didn't get the same xorsum as python. 0x%0.16x 0x%0.16x" % (ullResult, expectedUllSum)
# print "%30s" % "hex(bitResult):", hex(ullResult)
h2o.check_sandbox_for_errors()
print "first result was from a sum. others are xorsum"
print "ullResultList:"
for ullResult, fpResult in ullResultList:
print "%30s" % "ullResult (0.16x):", "0x%0.16x %s" % (ullResult, fpResult)
print "%30s" % "expectedUllSum (0.16x):", "0x%0.16x %s" % (expectedUllSum, expectedUllSumAsDouble)
print "%30s" % "expectedFpSum (0.16x):", "0x%0.16x %s" % (expectedFpSumAsLongLong, expectedFpSum)
def test_exec2_xorsum(self):
h2o.beta_features = True
SYNDATASETS_DIR = h2o.make_syn_dir()
tryList = [
(ROWS, 1, 'r1', 0, 10, None),
]
for trial in range(10):
ullResultList = []
for (rowCount, colCount, hex_key, expectedMin, expectedMax,
expected) in tryList:
SEEDPERFILE = random.randint(0, sys.maxint)
# dynamic range of the data may be useful for estimating error
maxDelta = expectedMax - expectedMin
csvFilename = 'syn_real_' + str(rowCount) + 'x' + str(
colCount) + '.csv'
csvPathname = SYNDATASETS_DIR + '/' + csvFilename
csvPathnameFull = h2i.find_folder_and_filename(
None, csvPathname, returnFullPath=True)
print "Creating random", csvPathname
(expectedUllSum,
expectedFpSum) = write_syn_dataset(csvPathname, rowCount,
colCount, expectedMin,
expectedMax, SEEDPERFILE)
expectedUllSumAsDouble = h2o_util.unsignedLongLongToDouble(
expectedUllSum)
expectedFpSumAsLongLong = h2o_util.doubleToUnsignedLongLong(
expectedFpSum)
parseResult = h2i.import_parse(path=csvPathname,
schema='put',
hex_key=hex_key,
timeoutSecs=3000,
retryDelaySecs=2)
inspect = h2o_cmd.runInspect(key=hex_key)
print "numRows:", inspect['numRows']
print "numCols:", inspect['numCols']
inspect = h2o_cmd.runInspect(key=hex_key, offset=-1)
print "inspect offset = -1:", h2o.dump_json(inspect)
# looking at the 8 bytes of bits for the h2o doubles
# xorsum will zero out the sign and exponent
for execExpr in exprList:
for r in range(10):
start = time.time()
(execResult, fpResult) = h2e.exec_expr(h2o.nodes[0],
execExpr,
resultKey='h',
timeoutSecs=300)
print r, 'exec took', time.time() - start, 'seconds'
print r, "execResult:", h2o.dump_json(execResult)
h2o_cmd.runStoreView()
ullResult = h2o_util.doubleToUnsignedLongLong(fpResult)
ullResultList.append((ullResult, fpResult))
print "%30s" % "ullResult (0.16x):", "0x%0.16x %s" % (
ullResult, fpResult)
print "%30s" % "expectedUllSum (0.16x):", "0x%0.16x %s" % (
expectedUllSum, expectedUllSumAsDouble)
print "%30s" % "expectedFpSum (0.16x):", "0x%0.16x %s" % (
expectedFpSumAsLongLong, expectedFpSum)
# allow diff of the lsb..either way
# if ullResult!=expectedUllSum and abs((ullResult-expectedUllSum)>3):
if ullResult != expectedUllSum:
raise Exception(
"h2o didn't get the same xorsum as python. 0x%0.16x 0x%0.16x"
% (ullResult, expectedUllSum))
print "h2o didn't get the same xorsum as python. 0x%0.16x 0x%0.16x" % (
ullResult, expectedUllSum)
h2o.check_sandbox_for_errors()
print "first result was from a sum. others are xorsum"
print "ullResultList:"
for ullResult, fpResult in ullResultList:
print "%30s" % "ullResult (0.16x):", "0x%0.16x %s" % (
ullResult, fpResult)
print "%30s" % "expectedUllSum (0.16x):", "0x%0.16x %s" % (
expectedUllSum, expectedUllSumAsDouble)
print "%30s" % "expectedFpSum (0.16x):", "0x%0.16x %s" % (
expectedFpSumAsLongLong, expectedFpSum)
def test_exec2_xorsum2(self):
SYNDATASETS_DIR = h2o.make_syn_dir()
tryList = [
(ROWS, 1, 'r1', 0, 10, None),
]
for trial in range(3):
ullResultList = []
NUM_FORMAT_CASES = h2o_util.fp_format()
for (rowCount, colCount, hex_key, expectedMin, expectedMax,
expected) in tryList:
SEEDPERFILE = random.randint(0, sys.maxint)
# dynamic range of the data may be useful for estimating error
maxDelta = expectedMax - expectedMin
csvFilename = 'syn_real_' + str(rowCount) + 'x' + str(
colCount) + '.csv'
csvPathname = SYNDATASETS_DIR + '/' + csvFilename
csvPathnameFull = h2i.find_folder_and_filename(
None, csvPathname, returnFullPath=True)
print "Creating random", csvPathname
sel = random.randint(0, NUM_FORMAT_CASES - 1)
(expectedUllSum, expectedFpSum) = write_syn_dataset(
csvPathname, rowCount, colCount, expectedMin, expectedMax,
SEEDPERFILE, sel)
expectedUllSumAsDouble = h2o_util.unsignedLongLongToDouble(
expectedUllSum)
expectedFpSumAsLongLong = h2o_util.doubleToUnsignedLongLong(
expectedFpSum)
parseResult = h2i.import_parse(path=csvPathname,
schema='put',
hex_key=hex_key,
timeoutSecs=3000,
retryDelaySecs=2)
inspect = h2o_cmd.runInspect(key=hex_key)
print "numRows:", inspect['numRows']
print "numCols:", inspect['numCols']
inspect = h2o_cmd.runInspect(key=hex_key, offset=-1)
print "inspect offset = -1:", h2o.dump_json(inspect)
# looking at the 8 bytes of bits for the h2o doubles
# xorsum will zero out the sign and exponent
for execExpr in exprList:
for repeate in range(3):
start = time.time()
(execResult, fpResult) = h2e.exec_expr(h2o.nodes[0],
execExpr,
resultKey=None,
timeoutSecs=300)
print 'exec took', time.time() - start, 'seconds'
print "execResult:", h2o.dump_json(execResult)
ullResult = h2o_util.doubleToUnsignedLongLong(fpResult)
ullResultList.append((ullResult, fpResult))
print "%30s" % "ullResult (0.16x):", "0x%0.16x %s" % (
ullResult, fpResult)
print "%30s" % "expectedUllSum (0.16x):", "0x%0.16x %s" % (
expectedUllSum, expectedUllSumAsDouble)
print "%30s" % "expectedFpSum (0.16x):", "0x%0.16x %s" % (
expectedFpSumAsLongLong, expectedFpSum)
# allow diff of the lsb..either way. needed when integers are parsed
# okay for a couple of lsbs to be wrong, due to conversion from stringk
# ullResult (0.16x): 0x02c1a21f923cee96 2.15698793923e-295
# expectedUllSum (0.16x): 0x02c1a21f923cee97 2.15698793923e-295
# expectedFpSum (0.16x): 0x42f054af32b3c408 2.87294442126e+14
# ullResult and expectedUllSum are Q ints, (64-bit) so can subtract them.
# I guess we don't even care about sign, since we zero the first 4 bits (xorsum) to avoid nan/inf issues
if ullResult != expectedUllSum and (
abs(ullResult - expectedUllSum) >
ALLOWED_DELTA):
emsg = "h2o didn't get the same xorsum as python. 0x%0.16x 0x%0.16x" % (
ullResult, expectedUllSum)
if STOP_ON_ERROR:
raise Exception(emsg)
else:
print emsg
# print "%30s" % "hex(bitResult):", hex(ullResult)
h2o.check_sandbox_for_errors()
print "first result was from a sum. others are xorsum"
print "ullResultList:"
for ullResult, fpResult in ullResultList:
print "%30s" % "ullResult (0.16x):", "0x%0.16x %s" % (
ullResult, fpResult)
print "%30s" % "expectedUllSum (0.16x):", "0x%0.16x %s" % (
expectedUllSum, expectedUllSumAsDouble)
print "%30s" % "expectedFpSum (0.16x):", "0x%0.16x %s" % (
expectedFpSumAsLongLong, expectedFpSum)
def test_exec2_xorsum(self):
SYNDATASETS_DIR = h2o.make_syn_dir()
tryList = [
(ROWS, 1, 'r1', 0, 10, None),
]
for trial in range(10):
ullResultList = []
for (rowCount, colCount, hex_key, expectedMin, expectedMax, expected) in tryList:
SEEDPERFILE = random.randint(0, sys.maxint)
# dynamic range of the data may be useful for estimating error
maxDelta = expectedMax - expectedMin
csvFilename = 'syn_real_' + str(rowCount) + 'x' + str(colCount) + '.csv'
csvPathname = SYNDATASETS_DIR + '/' + csvFilename
csvPathnameFull = h2i.find_folder_and_filename(None, csvPathname, returnFullPath=True)
print "Creating random", csvPathname
(expectedUllSum, expectedFpSum) = write_syn_dataset(csvPathname,
rowCount, colCount, expectedMin, expectedMax, SEEDPERFILE)
expectedUllSumAsDouble = h2o_util.unsignedLongLongToDouble(expectedUllSum)
expectedFpSumAsLongLong = h2o_util.doubleToUnsignedLongLong(expectedFpSum)
parseResult = h2i.import_parse(path=csvPathname, schema='put', hex_key=hex_key,
timeoutSecs=3000, retryDelaySecs=2)
numRows, numCols, parse_key = h2o_cmd.infoFromParse(parseResult)
assert parse_key == hex_key
assert numCols == colCount
assert numRows == rowCount
inspect = h2o_cmd.runInspect(key=hex_key)
missingList, labelList, numRows, numCols = h2o_cmd.infoFromInspect(inspect)
assert len(missingList) == 0
# looking at the 8 bytes of bits for the h2o doubles
# xorsum will zero out the sign and exponent
for execExpr in exprList:
for r in range(10):
if 1==0:
execResult = h2o_cmd.runExec(ast=execExpr, timeoutSecs=30)
fpResult = execResult['scalar']
else:
(execResult, fpResult) = h2e.exec_expr(h2o.nodes[0], execExpr, resultKey='x', timeoutSecs=300)
# print dump_json(h2o.n0.frames(key="h"))
# (execResult, fpResult) = h2e.exec_expr(h2o.nodes[0], execExpr, resultKey='h', timeoutSecs=300)
# print dump_json(h2o.n0.frames(key="r1"))
print r, "execResult:", h2o.dump_json(execResult)
h2o_cmd.runStoreView()
ullResult = h2o_util.doubleToUnsignedLongLong(fpResult)
ullResultList.append((ullResult, fpResult))
print "%30s" % "ullResult (0.16x):", "0x%0.16x %s" % (ullResult, fpResult)
print "%30s" % "expectedUllSum (0.16x):", "0x%0.16x %s" % (expectedUllSum, expectedUllSumAsDouble)
print "%30s" % "expectedFpSum (0.16x):", "0x%0.16x %s" % (expectedFpSumAsLongLong, expectedFpSum)
# allow diff of the lsb..either way
# if ullResult!=expectedUllSum and abs((ullResult-expectedUllSum)>3):
if ullResult!=expectedUllSum:
raise Exception("h2o didn't get the same xorsum as python. 0x%0.16x 0x%0.16x" % \
(ullResult, expectedUllSum))
print "h2o didn't get the same xorsum as python. 0x%0.16x 0x%0.16x" % \
(ullResult, expectedUllSum)
h2o.check_sandbox_for_errors()
print "first result was from a sum. others are xorsum"
print "ullResultList:"
for ullResult, fpResult in ullResultList:
print "%30s" % "ullResult (0.16x):", "0x%0.16x %s" % (ullResult, fpResult)
print "%30s" % "expectedUllSum (0.16x):", "0x%0.16x %s" % (expectedUllSum, expectedUllSumAsDouble)
print "%30s" % "expectedFpSum (0.16x):", "0x%0.16x %s" % (expectedFpSumAsLongLong, expectedFpSum)