def test1(self):
run_context = SuiteContext('unittest')
db_context = run_context.getDBContext('remote')
self.LOGGER = run_context.get_logger()
# Set up a test table definition
with get_temp_table(db_context) as table:
for i in xrange(100):
table.add(FieldSpec("col_{}".format(i), 'NVARCHAR', 8))
fts = FastTableStream(table, use_names=False, raw=True)
data = [[unicode(100 * j + i) for i in xrange(100)]
for j in xrange(1000)]
def do_write():
with fts:
for j in xrange(5):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(
run_context.logs_dir,
'fast_table_write_remote_unicode_raw_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(table))
def main():
"""
This runs a test of the complementary_merge wrapper/glue code.
"""
run_context = SuiteContext('OGT_test{}'.format(RUN_CONTEXT_NUMBER))
log = run_context.get_logger('ComplementaryMerge')
db_context = run_context.getDBContext()
intermediate_path = 'OGT Fall 2012' if (
SUMMER_OR_FALL
== 'F') else 'OGT Spring 2012' # Summer has SAS variables.
pathname = os.path.join(CVSROOT, 'CSSC Score Reporting', intermediate_path,
'Code/Development/Intake')
bookmap_location_file_name = os.path.join(pathname,
BOOKMAP_LOCATION_FILE_NAME)
log.debug("main - bookmap_location_file_name[%s]" %
bookmap_location_file_name)
print("bookmap_location_file_name[%s]" % bookmap_location_file_name)
mergespec_file_name = os.path.join(run_context.tests_safe_dir,
MERGESPEC_FILE_NAME)
input_table_names = {
FLAT_TABLE_KEY_NAME: 'rc2FINAL',
'C': 'mc_table_C',
'M': 'mc_table_M',
'R': 'mc_table_R',
'S': 'mc_table_S',
'W': 'mc_table_W'
}
output_table_names = {
FLAT_TABLE_KEY_NAME: 'rc2FINAL_cmrg',
'C': 'mc_table_C_cmrg',
'M': 'mc_table_M_cmrg',
'R': 'mc_table_R_cmrg',
'S': 'mc_table_S_cmrg',
'W': 'mc_table_W_cmrg'
}
for key in output_table_names:
dbutilities.drop_table_if_exists(db_context=db_context,
table=output_table_names[key])
try:
complementary_merge(
run_context=run_context,
bookmap_location_file_name=bookmap_location_file_name,
bookmap_sheet=BOOKMAP_SHEET,
mergespec_file_name=mergespec_file_name,
input_table_names=input_table_names,
output_table_names=output_table_names)
#create_mergespec_file( run_context=run_context, input_table_names=input_table_names,
# new_mergespec_file='C:/new_mergespec_file.csv' )
except Exception, error_msg:
log.exception('\n\n')
raise
class TestPreQC( unittest.TestCase ):
def setUp(self):
self.run_context = SuiteContext( 'unittest' )
self.db_context = self.run_context.getDBContext( 'unittest' )
self.LOGGER = self.run_context.get_logger()
def tearDown(self):
pass
def runTest(self):
pass
def main( ):
"""
This runs a test of the district_performance_summary_report wrapper/glue code.
"""
run_context = SuiteContext( 'OGT_test{}'.format( RUN_CONTEXT_NUMBER ) )
log = run_context.get_logger( 'DistrictPerformanceSummaryReport' )
specfile = os.path.join( run_context.tests_safe_dir, SPECFILE )
log.debug( "main - specfile[{}]".format( specfile ) )
# sch_type = H = 12 records
# sch_type = P = 10495 records
dpsr.district_performance_summary_report(run_context=run_context, specfile=specfile, input_table_name='student' )
class TestFastTableStream(unittest.TestCase):
def setUp(self):
self.run_context = SuiteContext('unittest')
self.db_context = self.run_context.getDBContext('unittest')
self.LOGGER = self.run_context.get_logger()
# Set up a test table definition
self.table = get_temp_table(self.db_context)
(self.table.add(FieldSpec("col1", "NVARCHAR", 8)).add(
FieldSpec("col2", "FLOAT")).add(FieldSpec("col3", "TINYINT")).add(
FieldSpec("col4", "INT")).add(FieldSpec("col5", "BIGINT")))
def tearDown(self):
self.table.drop()
def runTest(self):
pass
def testWriteUnicodeCharacters(self):
del self.table[:]
for i in xrange(100):
self.table.add(FieldSpec("col_{}".format(i), 'NVARCHAR', 8))
fts = FastTableStream(self.table, use_names=False)
with fts:
for j in xrange(10000):
fts.write([100 * j + i for i in xrange(100)])
self.LOGGER.info("Table name {}".format(self.table))
def testWriteUnicodeMany(self):
del self.table[:]
for i in xrange(100):
self.table.add(FieldSpec("col_{}".format(i), 'NVARCHAR', 8))
fts = FastTableStream(self.table, use_names=False)
data = [[100 * j + i for i in xrange(100)] for j in xrange(1000)]
def do_write():
with fts:
for j in xrange(20):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(self.run_context.logs_dir,
'fast_table_write_unicode_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(self.table))
def testWriteUnicodeRaw(self):
del self.table[:]
for i in xrange(100):
self.table.add(FieldSpec("col_{}".format(i), 'NVARCHAR', 8))
fts = FastTableStream(self.table, use_names=False, raw=True)
data = [[unicode(100 * j + i) for i in xrange(100)]
for j in xrange(1000)]
def do_write():
with fts:
for j in xrange(20):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(self.run_context.logs_dir,
'fast_table_write_unicode_raw_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(self.table))
def testWriteUnicodeNoNull(self):
del self.table[:]
for i in xrange(100):
self.table.add(
FieldSpec("col_{}".format(i), 'NVARCHAR', 8, nullable=False))
fts = FastTableStream(self.table, use_names=False, raw=True)
data = [[unicode(100 * j + i) for i in xrange(100)]
for j in xrange(1000)]
def do_write():
with fts:
for j in xrange(20):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(self.run_context.logs_dir,
'fast_table_write_unicode_nonull_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(self.table))
def testWriteCharacters(self):
del self.table[:]
for i in xrange(100):
self.table.add(FieldSpec("col_{}".format(i), 'VARCHAR', 8))
fts = FastTableStream(self.table, use_names=False)
with fts:
for j in xrange(10000):
fts.write([str(100 * j + i)[:8] for i in xrange(100)])
self.LOGGER.info("Table name {}".format(self.table))
def testWriteMany(self):
del self.table[:]
for i in xrange(100):
self.table.add(FieldSpec("col_{}".format(i), 'VARCHAR', 8))
fts = FastTableStream(self.table, use_names=False)
data = [[100 * j + i for i in xrange(100)] for j in xrange(1000)]
def do_write():
with fts:
for j in xrange(20):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(self.run_context.logs_dir,
'fast_table_write_char_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(self.table))
def testWriteRaw(self):
del self.table[:]
for i in xrange(100):
self.table.add(FieldSpec("col_{}".format(i), 'VARCHAR', 8))
fts = FastTableStream(self.table, use_names=False, raw=True)
data = [[str(100 * j + i) for i in xrange(100)] for j in xrange(1000)]
def do_write():
with fts:
for j in xrange(20):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(self.run_context.logs_dir,
'fast_table_write_char_raw_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(self.table))
def testWriteIntegerMany(self):
del self.table[:]
for i in xrange(100):
self.table.add(FieldSpec("col_{}".format(i), 'INT'))
fts = FastTableStream(self.table, use_names=False)
data = [[100 * j + i for i in xrange(100)] for j in xrange(1000)]
def do_write():
with fts:
for j in xrange(20):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(
self.run_context.logs_dir,
'fast_table_write_int_with_checks_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(self.table))
def testWriteIntegerRaw(self):
del self.table[:]
for i in xrange(100):
self.table.add(FieldSpec("col_{}".format(i), 'INT'))
fts = FastTableStream(self.table, use_names=False, raw=True)
data = [[100 * j + i for i in xrange(100)] for j in xrange(1000)]
def do_write():
with fts:
for j in xrange(20):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(self.run_context.logs_dir,
'fast_table_write_int_raw_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(self.table))
def testWriteNoNull(self):
del self.table[:]
for i in xrange(100):
self.table.add(
FieldSpec("col_{}".format(i), 'VARCHAR', 8, nullable=False))
fts = FastTableStream(self.table, use_names=False, raw=True)
data = [[str(100 * j + i) for i in xrange(100)] for j in xrange(1000)]
def do_write():
with fts:
for j in xrange(20):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(self.run_context.logs_dir,
'fast_table_write_char_nonull_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(self.table))
def testWriteFloatNoNull(self):
del self.table[:]
for i in xrange(100):
self.table.add(
FieldSpec("col_{}".format(i), 'FLOAT', nullable=False))
fts = FastTableStream(self.table,
use_names=False,
raw=True,
dumpfile="C:\\Scratch\\float_no_null.dat")
data = [[100.0 * j + i for i in xrange(100)] for j in xrange(1000)]
def do_write():
with fts:
for j in xrange(20):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(self.run_context.logs_dir,
'fast_table_write_float_nonull_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(self.table))
def testWriteFloatMany(self):
del self.table[:]
for i in xrange(100):
self.table.add(FieldSpec("col_{}".format(i), 'FLOAT'))
fts = FastTableStream(self.table,
use_names=False,
dumpfile="C:\\Scratch\\float.dat")
data = [[100.0 * j + i for i in xrange(100)] for j in xrange(1000)]
def do_write():
with fts:
for j in xrange(20):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(self.run_context.logs_dir,
'fast_table_write_float_nonull_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(self.table))
def testWriteMixed(self):
del self.table[:]
for i in xrange(33):
self.table.add(FieldSpec("float_{}".format(i), 'FLOAT'))
self.table.add(FieldSpec("int_{}".format(i), 'INT'))
self.table.add(FieldSpec("str_{}".format(i), 'VARCHAR', 6))
fts = FastTableStream(self.table, use_names=False)
data = []
for j in xrange(1000):
row = []
for i in xrange(33):
k = 100 * j + i
row.extend((int(k), float(k), str(k)))
def do_write():
with fts:
for j in xrange(20):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(self.run_context.logs_dir,
'fast_table_write_mixed_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(self.table))
def testValidateNewTable(self):
drop_table_if_exists(self.table)
table_stream = FastTableStream(self.table)
table_stream.validate_write_inputs()
def testValidateFailNewEmptyTable(self):
drop_table_if_exists(self.table)
del self.table[:]
table_stream = FastTableStream(self.table)
try:
table_stream.validate_write_inputs()
except ValueError:
# Expected error
return
self.fail(
"Expected a ValueError if called on to create a new table with no fields"
)
def testValidateExistingTable(self):
drop_table_if_exists(self.table)
self.db_context.executeNoResults(self.table.definition)
table_stream = FastTableStream(self.table)
table_stream.validate_write_inputs()
def testValidateExistingTableAgainstEmptySpec(self):
drop_table_if_exists(self.table)
self.db_context.executeNoResults(self.table.definition)
del self.table[:]
table_stream = FastTableStream(self.table)
table_stream.validate_write_inputs()
def testValidateFailWrongColumnCount(self):
drop_table_if_exists(self.table)
self.db_context.executeNoResults(self.table.definition)
self.table.add(FieldSpec('col7', 'NVARCHAR', 15))
table_stream = FastTableStream(self.table)
try:
table_stream.validate_write_inputs()
except ValueError:
# Expected error
return
self.fail(
"Expected a ValueError if TableSpec has different column count from db"
)
def testValidateFailWrongColumnName(self):
drop_table_if_exists(self.table)
self.db_context.executeNoResults(self.table.definition)
col1 = self.table.pop('col1')
col1.field_name = 'col_new'
self.table.add(col1)
table_stream = FastTableStream(self.table)
try:
table_stream.validate_write_inputs()
except ValueError:
# Expected error
return
self.fail("Expected a ValueError if column names do not match")
def testValidateFailWrongColumnType(self):
drop_table_if_exists(self.table)
self.db_context.executeNoResults(self.table.definition)
self.table['col1'].basic_type = 'VARCHAR'
table_stream = FastTableStream(self.table)
try:
table_stream.validate_write_inputs()
except ValueError:
# Expected error
return
self.fail(
"Expected a ValueError if db column type is different from TableSpec column type"
)
def testValidateFailColumnTooShort(self):
drop_table_if_exists(self.table)
self.db_context.executeNoResults(self.table.definition)
self.table['col1'].data_length = 100
table_stream = FastTableStream(self.table)
try:
table_stream.validate_write_inputs()
except ValueError:
# Expected error
return
self.fail(
"Expected a ValueError if db column is shorter than TableSpec column"
)
def testValidateColumnLonger(self):
drop_table_if_exists(self.table)
self.db_context.executeNoResults(self.table.definition)
self.table['col1'].data_length = 1
table_stream = FastTableStream(self.table)
table_stream.validate_write_inputs()