def setUpClass(cls):
cls.runContext = SuiteContext('unittest')
cls.db_context = cls.runContext.getDBContext(tag='unittest')
clear_all(cls.db_context)
cls.testDataDir = os.path.join(cls.runContext.tests_safe_dir,
"peercompare_test")
#setup the tables we will need in the database
reader = SafeExcelReader(run_context=cls.runContext,
db_context=cls.db_context,
filename=os.path.join(cls.testDataDir,
_SAS_FILE),
output_table=_SIMILAR_TABLE,
delimiter='|')
reader.createTable()
reader = SafeExcelReader(run_context=cls.runContext,
db_context=cls.db_context,
filename=os.path.join(cls.testDataDir,
_DATA_FILE),
output_table=_DATA_TABLE)
reader.createTable()
reader = SafeExcelReader(run_context=cls.runContext,
db_context=cls.db_context,
filename=os.path.join(cls.testDataDir,
_AGG_FILE),
sheet_name='peerCompare',
output_table=_AGG_TABLE)
reader.createTable()
def setUp(self):
self.runContext = SuiteContext('unittest')
self.db_context = self.runContext.getDBContext(tag='unittest')
clear_all(self.db_context)
self.reader = SafeExcelReader(self.runContext)
self.reader.db_context = self.db_context
self.testDataDir = self.runContext.tests_safe_dir
def test1(self):
run_context = SuiteContext('unittest')
db_context = run_context.getDBContext('remote')
self.LOGGER = run_context.get_logger()
# Set up a test table definition
with get_temp_table(db_context) as table:
for i in xrange(100):
table.add(FieldSpec("col_{}".format(i), 'NVARCHAR', 8))
fts = FastTableStream(table, use_names=False, raw=True)
data = [[unicode(100 * j + i) for i in xrange(100)]
for j in xrange(1000)]
def do_write():
with fts:
for j in xrange(5):
fts.write_many(data)
pr = cProfile.Profile()
pr.enable()
pr.runcall(do_write)
pr.disable()
filename = os.path.join(
run_context.logs_dir,
'fast_table_write_remote_unicode_raw_profile.txt')
with open(filename, 'w') as f:
stats = pstats.Stats(pr, stream=f)
stats.print_stats()
self.LOGGER.info("Table name {}".format(table))
def setUpClass(cls):
cls.runContext = SuiteContext('unittest')
cls.db_context = cls.runContext.getDBContext(tag='unittest')
clear_all(cls.db_context)
cls.testDataDir = os.path.join(cls.runContext.tests_safe_dir,
"erasure_test")
cls.bm_testDataDir = os.path.join(cls.runContext.tests_safe_dir,
"bookmapreader_test")
#setup the tables we will need in the database
reader = SafeExcelReader(run_context=cls.runContext,
db_context=cls.db_context,
buffer_size=100,
scan_all=True,
filename=os.path.join(cls.testDataDir,
_DATAFILE_1),
output_table='AIR1',
range_=(0, 0, 500, 1024))
reader.createTable()
reader = SafeExcelReader(run_context=cls.runContext,
db_context=cls.db_context,
buffer_size=100,
scan_all=True,
filename=os.path.join(cls.testDataDir,
_DATAFILE_2),
output_table='AIR2',
range_=(0, 0, 500, 1024))
reader.createTable()
def setUpClass(cls):
cls.runContext = SuiteContext('unittest')
cls.db_context = cls.runContext.getDBContext(tag='unittest')
clear_all(cls.db_context)
cls.testDataDir = os.path.join(cls.runContext.tests_safe_dir,
"means_test")
#setup the tables we will need in the database
reader = SafeExcelReader(run_context=cls.runContext,
db_context=cls.db_context,
filename=os.path.join(cls.testDataDir,
_XLS_FILE),
sheet_name=_MEANS_SHEET,
output_table=_MEANS_SHEET)
reader.createTable()
reader = SafeExcelReader(run_context=cls.runContext,
db_context=cls.db_context,
filename=os.path.join(cls.testDataDir,
_XLS_FILE),
sheet_name=_PERCENTS_SHEET,
output_table=_PERCENTS_SHEET)
reader.createTable()
reader = SafeExcelReader(run_context=cls.runContext,
db_context=cls.db_context,
filename=os.path.join(cls.testDataDir,
_DATA_FILE),
scan_all=True,
delimiter='|',
buffer_size=100,
output_table='studentg3_n')
reader.createTable()
def setUp(self):
self.run_context = SuiteContext( "unittest" )
self.db_context = self.run_context.getDBContext( "unittest" )
self.static_context = self.run_context.getDBContext( "static" )
self.answer_dir = os.path.join( self.run_context.logs_dir, 'ttest_level_tests' )
if not os.path.exists( self.answer_dir ):
os.makedirs( self.answer_dir )
self.data_dir = os.path.join( self.run_context.tests_safe_dir,
'ttest', 'input_data' )
self.specimen_dir = os.path.join( self.run_context.tests_safe_dir,
'ttest', 'sas_outputs' )
# libname ttest "H:\share\CSSC Folder\Score Report Group\Test Data\lib_TTestLevel";
# %let agg_file = &cvsroot.\ScoreReportMacros\UnitTested\lib_TTestLevel\test\HI Spring 2008 Aggregations_Melissa.xls;
# %let sheet=ttestlevel;
# %SafeExcelRead(filename=&agg_file., sheetname =&sheet., DS_out =aggds);
self.run_context.debug( "Reading data for ttest_level tests" )
agg_file = os.path.join( self.data_dir, _AGG_FILE)
reader = SafeExcelReader( self.run_context,
agg_file, "ttestlevel", scan_all = True )
self.agg_ds = [ row for row in reader.getRows() ]
# Import the input datasets
reader.db_context = self.static_context
for ( grade, filename, table_name, sheet_name ) in _GRADE_FILES:
if not table_exists( table_name, self.static_context ):
self.run_context.debug( "Reading data for grade {}".format( grade ) )
reader.filename = os.path.join( self.data_dir, filename )
reader.outputTable = table_name
reader.sheetName = sheet_name
reader.createTable()
def setUp(self):
self.run_context = SuiteContext('unittest')
self.out_dir = os.path.join(self.run_context.tests_dir, 'merge')
if os.path.exists(self.out_dir):
shutil.rmtree(self.out_dir)
os.makedirs(self.out_dir)
self.db_context = self.run_context.getDBContext('unittest')
self.setUpData()
self.setUpMerge()
def setUp(self):
self.run_context = SuiteContext('unittest')
self.db_context = self.run_context.getDBContext('unittest')
self.data_dir = os.path.join(self.run_context.tests_safe_dir, 'ttest',
'input_data')
self.answer_dir = os.path.join(self.run_context.logs_dir,
'ttest_tests')
self.specimen_dir = os.path.join(self.run_context.tests_safe_dir,
'ttest', 'sas_outputs')
def main():
"""
This runs a test of the complementary_merge wrapper/glue code.
"""
run_context = SuiteContext('OGT_test{}'.format(RUN_CONTEXT_NUMBER))
log = run_context.get_logger('ComplementaryMerge')
db_context = run_context.getDBContext()
intermediate_path = 'OGT Fall 2012' if (
SUMMER_OR_FALL
== 'F') else 'OGT Spring 2012' # Summer has SAS variables.
pathname = os.path.join(CVSROOT, 'CSSC Score Reporting', intermediate_path,
'Code/Development/Intake')
bookmap_location_file_name = os.path.join(pathname,
BOOKMAP_LOCATION_FILE_NAME)
log.debug("main - bookmap_location_file_name[%s]" %
bookmap_location_file_name)
print("bookmap_location_file_name[%s]" % bookmap_location_file_name)
mergespec_file_name = os.path.join(run_context.tests_safe_dir,
MERGESPEC_FILE_NAME)
input_table_names = {
FLAT_TABLE_KEY_NAME: 'rc2FINAL',
'C': 'mc_table_C',
'M': 'mc_table_M',
'R': 'mc_table_R',
'S': 'mc_table_S',
'W': 'mc_table_W'
}
output_table_names = {
FLAT_TABLE_KEY_NAME: 'rc2FINAL_cmrg',
'C': 'mc_table_C_cmrg',
'M': 'mc_table_M_cmrg',
'R': 'mc_table_R_cmrg',
'S': 'mc_table_S_cmrg',
'W': 'mc_table_W_cmrg'
}
for key in output_table_names:
dbutilities.drop_table_if_exists(db_context=db_context,
table=output_table_names[key])
try:
complementary_merge(
run_context=run_context,
bookmap_location_file_name=bookmap_location_file_name,
bookmap_sheet=BOOKMAP_SHEET,
mergespec_file_name=mergespec_file_name,
input_table_names=input_table_names,
output_table_names=output_table_names)
#create_mergespec_file( run_context=run_context, input_table_names=input_table_names,
# new_mergespec_file='C:/new_mergespec_file.csv' )
except Exception, error_msg:
log.exception('\n\n')
raise
def setUp(self):
self.runContext = SuiteContext('unittest')
self.db_context = self.runContext.getDBContext(tag='unittest')
clear_all(self.db_context)
self.reader = SafeExcelReader(self.runContext)
self.reader.db_context = self.db_context
self.testDataDir = self.runContext.tests_safe_dir
self.reader.filename = os.path.join(self.testDataDir, _XLS_FILE)
self.reader.sheetName = "Data1"
self.reader.outputTable = "Data1"
self.reader.createTable()
def setUp(self):
self.run_context = SuiteContext('unittest')
self.db_context = self.run_context.getDBContext('unittest')
self.LOGGER = self.run_context.get_logger()
# Set up a test table definition
self.table = get_temp_table(self.db_context)
(self.table.add(FieldSpec("col1", "NVARCHAR", 8)).add(
FieldSpec("col2", "FLOAT")).add(FieldSpec("col3", "TINYINT")).add(
FieldSpec("col4", "INT")).add(FieldSpec("col5", "BIGINT")))
def setUpClass(cls):
"""
Set up class variable 'context' for testing environment.
Note that the export_test.ini file is in the same directory as this test script so they can be coordinated within the project.
Future: SuiteContext() could accept absolute pathname of ini file and calculate and expose realpath of ini file
For now, can manually copy the current directory's export_test.ini file to a directory where it will be sought by RunContext, eg ~/air_python/export_test.ini, but it's a step we could otherwise skip.
"""
# Consider to add realpath and dirname to SuiteContext
cls.dirname = os.path.dirname(os.path.realpath(__file__))
cls.context = SuiteContext("export_test")
def main( ):
"""
This runs a test of the district_performance_summary_report wrapper/glue code.
"""
run_context = SuiteContext( 'OGT_test{}'.format( RUN_CONTEXT_NUMBER ) )
log = run_context.get_logger( 'DistrictPerformanceSummaryReport' )
specfile = os.path.join( run_context.tests_safe_dir, SPECFILE )
log.debug( "main - specfile[{}]".format( specfile ) )
# sch_type = H = 12 records
# sch_type = P = 10495 records
dpsr.district_performance_summary_report(run_context=run_context, specfile=specfile, input_table_name='student' )
def setUpClass(cls):
cls.runContext = SuiteContext('unittest')
cls.db_context = cls.runContext.getDBContext(tag='unittest')
clear_all(cls.db_context)
cls.testDataDir = os.path.join(cls.runContext.tests_safe_dir,
"converttopdf_test")
# remove the files we will output if they already exist
if os.path.exists(os.path.join(cls.testDataDir, _SQL_PDF_OUTPUT)):
os.remove(os.path.join(cls.testDataDir, _SQL_PDF_OUTPUT))
if os.path.exists(
os.path.join(cls.testDataDir, _XLS_PDF_OUTPUT_NOSTRETCH)):
os.remove(os.path.join(cls.testDataDir, _XLS_PDF_OUTPUT_NOSTRETCH))
if os.path.exists(
os.path.join(cls.testDataDir, _XLS_PDF_OUTPUT_STRETCH)):
os.remove(os.path.join(cls.testDataDir, _XLS_PDF_OUTPUT_STRETCH))
def setUpClass(cls):
cls.runContext = SuiteContext('unittest')
cls.db_context = cls.runContext.getDBContext(tag='unittest')
clear_all(cls.db_context)
cls.testDataDir = os.path.join(cls.runContext.tests_safe_dir,
"bookmapreader_test")
#setup the tables we will need in the database
reader = SafeExcelReader(run_context=cls.runContext,
db_context=cls.db_context,
filename=os.path.join(cls.testDataDir,
_XLS_FILE),
sheet_name=_SHEET_NAME,
output_table='bookmaplocations')
reader.createTable()
cnt = 0
for row in reader.getRows():
tablename = row['Subject'] + '_' + row['Form_Values']
filename_ = row["location"]
bm_reader = SafeExcelReader(run_context=cls.runContext,
db_context=cls.db_context,
filename=filename_,
sheet_name=_SHEET_NAME,
output_table=tablename)
bm_reader.createTable()
cnt += 1
#set the 'location' column to be the name of the table we created
cls.db_context.executeNoResults("""
UPDATE bookmaplocations
set location='{0}'
where subject='{1}' and form_values='{2}'
""".format(tablename, row['subject'], row['form_values']))
#get a list of table names to check
cls.tables = []
for row in cls.db_context.executeBuffered(
"SELECT [subject] + '_' + [form_values] from bookmaplocations"
):
cls.tables.append(row[0])
def setUp(self):
self.run_context = SuiteContext("unittest")
self.db_context = self.run_context.getDBContext("unittest")
self.static_context = self.run_context.getDBContext("static")
self.source_data_dir = self.run_context.getConfigFile(
"TESTS", "id_generator_test_source_data_dir",
"%(tests_safe_dir)s/id_generator_test/source_data")
if not table_exists('student_g3', self.static_context):
source_file = os.path.join(self.source_data_dir, _XLS_FILE)
reader = SafeExcelReader(self.run_context,
source_file,
"Sheet1",
'student_g3',
self.static_context,
scan_all=True)
reader.createTable()
self.answer_dir = os.path.join(self.run_context.logs_dir,
'id_generator_test')
if not os.path.exists(self.answer_dir):
os.makedirs(self.answer_dir)
self.specimen_dir = os.path.join(self.run_context.tests_safe_dir,
'id_generator_test', 'sas_outputs')
dbcontext.executeNoResults(
"sp_RENAME 'SCHOOL_ASCII.GRADE_RECODED','GRADE','COLUMN'")
dbcontext.executeNoResults(
"sp_RENAME 'SCHOOL_ASCII.GRADE_R_RECODED','RGRADE','COLUMN'")
@time_taken
def populate_school_intervention(dbcontext):
print " Create SCHOOL_INTERVENTION_ASCII"
drop_table_if_exists('SCHOOL_INTERVENTION_ASCII', dbcontext)
dbcontext.executeNoResults(sqls['SCHOOL_INTERVENTION'])
dbcontext.executeNoResults(
"ALTER TABLE SCHOOL_INTERVENTION_ASCII DROP COLUMN DCRXID")
dbcontext.executeNoResults(
"sp_RENAME 'SCHOOL_INTERVENTION_ASCII.DCRXID_RECODED','DCRXID','COLUMN'"
)
if __name__ == '__main__':
runcontext = SuiteContext('sharedtest')
dbcontext = runcontext.getDBContext()
filename_additional_label = 'C:\Projects\OGT_S12\OGT Spring 2012 Addresses & Labels 4_24_2012.xlsx'
filename_size_selection = 'C:\Projects\OGT_S12\OGT Spring 2012 Addresses & Labels 4_24_2012.xlsx'
starttime = time.time()
#ac = AsciiWriter(dbcontext = dbcontext,runcontext = runcontext, filename_additional_label = filename_additional_label, filename_size_selection = filename_size_selection)
ac = AsciiWriter(dbcontext=dbcontext, runcontext=runcontext)
ac.process()
endtime = time.time()
print 'TOTAL TIME TAKEN = ', endtime - starttime
def _process_keep_columns(self):
print '_process_keep_columns'
if len(self.keep_columns) <> 0:
dbcontext6 = self.rc.getDBContext(cached=False)
all_cols_qry = """Select name from sys.columns where object_id = OBJECT_ID('{tablename}')""".format(
tablename=self.output_table)
all_cols = dbcontext6.execute(all_cols_qry)
all_cols = [each[0].encode('ascii').upper() for each in all_cols]
drop_cols = [
each for each in all_cols if each not in self.keep_columns
]
for each in drop_cols:
DROP_QUERY = "ALTER TABLE {tablename} DROP COLUMN {columnname}".format(
tablename=self.output_table, columnname=each)
print DROP_QUERY
dbcontext6.executeNoResults(DROP_QUERY)
dbcontext6.close()
if __name__ == '__main__':
runcontext = SuiteContext('unittest')
dbcontext = runcontext.getDBContext(cached=False)
sm = SasMerge(dbcontext=dbcontext,
runcontext=runcontext,
merge_table_1="Employee_merge_1",
merge_table_2="Employee_merge_2",
mergeids=["Emp_Num"],
drop_columns=[],
keep_columns=['EMP_AGE'],
output_table="Merge_Output_table")
sm.process()
def setUp(self):
self.run_context = SuiteContext( 'unittest' )
self.db_context = self.run_context.getDBContext( 'unittest' )
self.LOGGER = self.run_context.get_logger()
def setUpClass(cls):
cls.runContext = SuiteContext('unittest')
cls.db_context = cls.runContext.getDBContext(tag='unittest')
clear_all(cls.db_context)
cls.testDataDir = os.path.join(cls.runContext.tests_safe_dir,
"intake_test")
create_index = "Create Clustered Index {indexname} on {tablename} ({fieldname})".format(
indexname='preqc_ix',
tablename=self.flat_tablename,
fieldname='id')
self.db.executeNoResults(create_index)
fields = 'flat_table_id, id'
for each in self.mc_items_table_names:
ixname = each + '_ix'
create_index = "CREATE CLUSTERED INDEX {indexname} on {tablename} ({fieldname})".format(
indexname=ixname.lower(), tablename=each, fieldname=fields)
self.db.executeNoResults(create_index)
if __name__ == '__main__':
from airassessmentreporting.testutility import SuiteContext
RC = SuiteContext('unittest')
# RC = RunContext('unittest')
dbcontext = RC.getDBContext()
print 'dbcontext=', dbcontext
x = PreQC(
runcontext=RC,
dbcontext=dbcontext,
# layoutfile='C:\CVS Projects\CSSC Score Reporting\OGT Fall 2012\Intake Layout\OGT_FA12_Op_DataLayout_IntakeLayout.xls',
layoutfile=
'C:\CVS Projects\CSSC Score Reporting\OGT Fall 2012\Intake Layout\OGT_FA12_Op_DataLayout_IntakeLayout.xls',
# inputfile='C:\SAS\OGT\Input\original-record1.txt',
inputfile=
'H:\\share\\Ohio Graduation Tests\\Technical\\2012 October\\ScoreReports\\TextFileFromDRC\\536215_2012OhioOGTFall_Regular.txt',
# inputfile='H:/share/Ohio Graduation Tests/Technical/2012 October/ScoreReports/TextFileFromDRC/536215_2012OhioOGTFall_Regular.txt',
# inputfile='H:\\share\\Ohio Graduation Tests\\Technical\\2012 July\\ScoreReports\\TextFileFromDRC\\536214_2012OhioOGTSummer_Regular.txt',
# inputfile='C:\SAS\OGT\Input\input-1.txt',
