def setUpClass(cls): cls.runContext = SuiteContext('unittest') cls.db_context = cls.runContext.getDBContext(tag='unittest') clear_all(cls.db_context) cls.testDataDir = os.path.join(cls.runContext.tests_safe_dir, "peercompare_test") #setup the tables we will need in the database reader = SafeExcelReader(run_context=cls.runContext, db_context=cls.db_context, filename=os.path.join(cls.testDataDir, _SAS_FILE), output_table=_SIMILAR_TABLE, delimiter='|') reader.createTable() reader = SafeExcelReader(run_context=cls.runContext, db_context=cls.db_context, filename=os.path.join(cls.testDataDir, _DATA_FILE), output_table=_DATA_TABLE) reader.createTable() reader = SafeExcelReader(run_context=cls.runContext, db_context=cls.db_context, filename=os.path.join(cls.testDataDir, _AGG_FILE), sheet_name='peerCompare', output_table=_AGG_TABLE) reader.createTable()
def setUp(self): self.runContext = SuiteContext('unittest') self.db_context = self.runContext.getDBContext(tag='unittest') clear_all(self.db_context) self.reader = SafeExcelReader(self.runContext) self.reader.db_context = self.db_context self.testDataDir = self.runContext.tests_safe_dir
def test1(self): run_context = SuiteContext('unittest') db_context = run_context.getDBContext('remote') self.LOGGER = run_context.get_logger() # Set up a test table definition with get_temp_table(db_context) as table: for i in xrange(100): table.add(FieldSpec("col_{}".format(i), 'NVARCHAR', 8)) fts = FastTableStream(table, use_names=False, raw=True) data = [[unicode(100 * j + i) for i in xrange(100)] for j in xrange(1000)] def do_write(): with fts: for j in xrange(5): fts.write_many(data) pr = cProfile.Profile() pr.enable() pr.runcall(do_write) pr.disable() filename = os.path.join( run_context.logs_dir, 'fast_table_write_remote_unicode_raw_profile.txt') with open(filename, 'w') as f: stats = pstats.Stats(pr, stream=f) stats.print_stats() self.LOGGER.info("Table name {}".format(table))
def setUpClass(cls): cls.runContext = SuiteContext('unittest') cls.db_context = cls.runContext.getDBContext(tag='unittest') clear_all(cls.db_context) cls.testDataDir = os.path.join(cls.runContext.tests_safe_dir, "erasure_test") cls.bm_testDataDir = os.path.join(cls.runContext.tests_safe_dir, "bookmapreader_test") #setup the tables we will need in the database reader = SafeExcelReader(run_context=cls.runContext, db_context=cls.db_context, buffer_size=100, scan_all=True, filename=os.path.join(cls.testDataDir, _DATAFILE_1), output_table='AIR1', range_=(0, 0, 500, 1024)) reader.createTable() reader = SafeExcelReader(run_context=cls.runContext, db_context=cls.db_context, buffer_size=100, scan_all=True, filename=os.path.join(cls.testDataDir, _DATAFILE_2), output_table='AIR2', range_=(0, 0, 500, 1024)) reader.createTable()
def setUpClass(cls): cls.runContext = SuiteContext('unittest') cls.db_context = cls.runContext.getDBContext(tag='unittest') clear_all(cls.db_context) cls.testDataDir = os.path.join(cls.runContext.tests_safe_dir, "means_test") #setup the tables we will need in the database reader = SafeExcelReader(run_context=cls.runContext, db_context=cls.db_context, filename=os.path.join(cls.testDataDir, _XLS_FILE), sheet_name=_MEANS_SHEET, output_table=_MEANS_SHEET) reader.createTable() reader = SafeExcelReader(run_context=cls.runContext, db_context=cls.db_context, filename=os.path.join(cls.testDataDir, _XLS_FILE), sheet_name=_PERCENTS_SHEET, output_table=_PERCENTS_SHEET) reader.createTable() reader = SafeExcelReader(run_context=cls.runContext, db_context=cls.db_context, filename=os.path.join(cls.testDataDir, _DATA_FILE), scan_all=True, delimiter='|', buffer_size=100, output_table='studentg3_n') reader.createTable()
def setUp(self): self.run_context = SuiteContext( "unittest" ) self.db_context = self.run_context.getDBContext( "unittest" ) self.static_context = self.run_context.getDBContext( "static" ) self.answer_dir = os.path.join( self.run_context.logs_dir, 'ttest_level_tests' ) if not os.path.exists( self.answer_dir ): os.makedirs( self.answer_dir ) self.data_dir = os.path.join( self.run_context.tests_safe_dir, 'ttest', 'input_data' ) self.specimen_dir = os.path.join( self.run_context.tests_safe_dir, 'ttest', 'sas_outputs' ) # libname ttest "H:\share\CSSC Folder\Score Report Group\Test Data\lib_TTestLevel"; # %let agg_file = &cvsroot.\ScoreReportMacros\UnitTested\lib_TTestLevel\test\HI Spring 2008 Aggregations_Melissa.xls; # %let sheet=ttestlevel; # %SafeExcelRead(filename=&agg_file., sheetname =&sheet., DS_out =aggds); self.run_context.debug( "Reading data for ttest_level tests" ) agg_file = os.path.join( self.data_dir, _AGG_FILE) reader = SafeExcelReader( self.run_context, agg_file, "ttestlevel", scan_all = True ) self.agg_ds = [ row for row in reader.getRows() ] # Import the input datasets reader.db_context = self.static_context for ( grade, filename, table_name, sheet_name ) in _GRADE_FILES: if not table_exists( table_name, self.static_context ): self.run_context.debug( "Reading data for grade {}".format( grade ) ) reader.filename = os.path.join( self.data_dir, filename ) reader.outputTable = table_name reader.sheetName = sheet_name reader.createTable()
def setUp(self): self.run_context = SuiteContext('unittest') self.out_dir = os.path.join(self.run_context.tests_dir, 'merge') if os.path.exists(self.out_dir): shutil.rmtree(self.out_dir) os.makedirs(self.out_dir) self.db_context = self.run_context.getDBContext('unittest') self.setUpData() self.setUpMerge()
def setUp(self): self.run_context = SuiteContext('unittest') self.db_context = self.run_context.getDBContext('unittest') self.data_dir = os.path.join(self.run_context.tests_safe_dir, 'ttest', 'input_data') self.answer_dir = os.path.join(self.run_context.logs_dir, 'ttest_tests') self.specimen_dir = os.path.join(self.run_context.tests_safe_dir, 'ttest', 'sas_outputs')
def main(): """ This runs a test of the complementary_merge wrapper/glue code. """ run_context = SuiteContext('OGT_test{}'.format(RUN_CONTEXT_NUMBER)) log = run_context.get_logger('ComplementaryMerge') db_context = run_context.getDBContext() intermediate_path = 'OGT Fall 2012' if ( SUMMER_OR_FALL == 'F') else 'OGT Spring 2012' # Summer has SAS variables. pathname = os.path.join(CVSROOT, 'CSSC Score Reporting', intermediate_path, 'Code/Development/Intake') bookmap_location_file_name = os.path.join(pathname, BOOKMAP_LOCATION_FILE_NAME) log.debug("main - bookmap_location_file_name[%s]" % bookmap_location_file_name) print("bookmap_location_file_name[%s]" % bookmap_location_file_name) mergespec_file_name = os.path.join(run_context.tests_safe_dir, MERGESPEC_FILE_NAME) input_table_names = { FLAT_TABLE_KEY_NAME: 'rc2FINAL', 'C': 'mc_table_C', 'M': 'mc_table_M', 'R': 'mc_table_R', 'S': 'mc_table_S', 'W': 'mc_table_W' } output_table_names = { FLAT_TABLE_KEY_NAME: 'rc2FINAL_cmrg', 'C': 'mc_table_C_cmrg', 'M': 'mc_table_M_cmrg', 'R': 'mc_table_R_cmrg', 'S': 'mc_table_S_cmrg', 'W': 'mc_table_W_cmrg' } for key in output_table_names: dbutilities.drop_table_if_exists(db_context=db_context, table=output_table_names[key]) try: complementary_merge( run_context=run_context, bookmap_location_file_name=bookmap_location_file_name, bookmap_sheet=BOOKMAP_SHEET, mergespec_file_name=mergespec_file_name, input_table_names=input_table_names, output_table_names=output_table_names) #create_mergespec_file( run_context=run_context, input_table_names=input_table_names, # new_mergespec_file='C:/new_mergespec_file.csv' ) except Exception, error_msg: log.exception('\n\n') raise
def setUp(self): self.runContext = SuiteContext('unittest') self.db_context = self.runContext.getDBContext(tag='unittest') clear_all(self.db_context) self.reader = SafeExcelReader(self.runContext) self.reader.db_context = self.db_context self.testDataDir = self.runContext.tests_safe_dir self.reader.filename = os.path.join(self.testDataDir, _XLS_FILE) self.reader.sheetName = "Data1" self.reader.outputTable = "Data1" self.reader.createTable()
def setUp(self): self.run_context = SuiteContext('unittest') self.db_context = self.run_context.getDBContext('unittest') self.LOGGER = self.run_context.get_logger() # Set up a test table definition self.table = get_temp_table(self.db_context) (self.table.add(FieldSpec("col1", "NVARCHAR", 8)).add( FieldSpec("col2", "FLOAT")).add(FieldSpec("col3", "TINYINT")).add( FieldSpec("col4", "INT")).add(FieldSpec("col5", "BIGINT")))
def setUpClass(cls): """ Set up class variable 'context' for testing environment. Note that the export_test.ini file is in the same directory as this test script so they can be coordinated within the project. Future: SuiteContext() could accept absolute pathname of ini file and calculate and expose realpath of ini file For now, can manually copy the current directory's export_test.ini file to a directory where it will be sought by RunContext, eg ~/air_python/export_test.ini, but it's a step we could otherwise skip. """ # Consider to add realpath and dirname to SuiteContext cls.dirname = os.path.dirname(os.path.realpath(__file__)) cls.context = SuiteContext("export_test")
def main( ): """ This runs a test of the district_performance_summary_report wrapper/glue code. """ run_context = SuiteContext( 'OGT_test{}'.format( RUN_CONTEXT_NUMBER ) ) log = run_context.get_logger( 'DistrictPerformanceSummaryReport' ) specfile = os.path.join( run_context.tests_safe_dir, SPECFILE ) log.debug( "main - specfile[{}]".format( specfile ) ) # sch_type = H = 12 records # sch_type = P = 10495 records dpsr.district_performance_summary_report(run_context=run_context, specfile=specfile, input_table_name='student' )
def setUpClass(cls): cls.runContext = SuiteContext('unittest') cls.db_context = cls.runContext.getDBContext(tag='unittest') clear_all(cls.db_context) cls.testDataDir = os.path.join(cls.runContext.tests_safe_dir, "converttopdf_test") # remove the files we will output if they already exist if os.path.exists(os.path.join(cls.testDataDir, _SQL_PDF_OUTPUT)): os.remove(os.path.join(cls.testDataDir, _SQL_PDF_OUTPUT)) if os.path.exists( os.path.join(cls.testDataDir, _XLS_PDF_OUTPUT_NOSTRETCH)): os.remove(os.path.join(cls.testDataDir, _XLS_PDF_OUTPUT_NOSTRETCH)) if os.path.exists( os.path.join(cls.testDataDir, _XLS_PDF_OUTPUT_STRETCH)): os.remove(os.path.join(cls.testDataDir, _XLS_PDF_OUTPUT_STRETCH))
def setUpClass(cls): cls.runContext = SuiteContext('unittest') cls.db_context = cls.runContext.getDBContext(tag='unittest') clear_all(cls.db_context) cls.testDataDir = os.path.join(cls.runContext.tests_safe_dir, "bookmapreader_test") #setup the tables we will need in the database reader = SafeExcelReader(run_context=cls.runContext, db_context=cls.db_context, filename=os.path.join(cls.testDataDir, _XLS_FILE), sheet_name=_SHEET_NAME, output_table='bookmaplocations') reader.createTable() cnt = 0 for row in reader.getRows(): tablename = row['Subject'] + '_' + row['Form_Values'] filename_ = row["location"] bm_reader = SafeExcelReader(run_context=cls.runContext, db_context=cls.db_context, filename=filename_, sheet_name=_SHEET_NAME, output_table=tablename) bm_reader.createTable() cnt += 1 #set the 'location' column to be the name of the table we created cls.db_context.executeNoResults(""" UPDATE bookmaplocations set location='{0}' where subject='{1}' and form_values='{2}' """.format(tablename, row['subject'], row['form_values'])) #get a list of table names to check cls.tables = [] for row in cls.db_context.executeBuffered( "SELECT [subject] + '_' + [form_values] from bookmaplocations" ): cls.tables.append(row[0])
def setUp(self): self.run_context = SuiteContext("unittest") self.db_context = self.run_context.getDBContext("unittest") self.static_context = self.run_context.getDBContext("static") self.source_data_dir = self.run_context.getConfigFile( "TESTS", "id_generator_test_source_data_dir", "%(tests_safe_dir)s/id_generator_test/source_data") if not table_exists('student_g3', self.static_context): source_file = os.path.join(self.source_data_dir, _XLS_FILE) reader = SafeExcelReader(self.run_context, source_file, "Sheet1", 'student_g3', self.static_context, scan_all=True) reader.createTable() self.answer_dir = os.path.join(self.run_context.logs_dir, 'id_generator_test') if not os.path.exists(self.answer_dir): os.makedirs(self.answer_dir) self.specimen_dir = os.path.join(self.run_context.tests_safe_dir, 'id_generator_test', 'sas_outputs')
dbcontext.executeNoResults( "sp_RENAME 'SCHOOL_ASCII.GRADE_RECODED','GRADE','COLUMN'") dbcontext.executeNoResults( "sp_RENAME 'SCHOOL_ASCII.GRADE_R_RECODED','RGRADE','COLUMN'") @time_taken def populate_school_intervention(dbcontext): print " Create SCHOOL_INTERVENTION_ASCII" drop_table_if_exists('SCHOOL_INTERVENTION_ASCII', dbcontext) dbcontext.executeNoResults(sqls['SCHOOL_INTERVENTION']) dbcontext.executeNoResults( "ALTER TABLE SCHOOL_INTERVENTION_ASCII DROP COLUMN DCRXID") dbcontext.executeNoResults( "sp_RENAME 'SCHOOL_INTERVENTION_ASCII.DCRXID_RECODED','DCRXID','COLUMN'" ) if __name__ == '__main__': runcontext = SuiteContext('sharedtest') dbcontext = runcontext.getDBContext() filename_additional_label = 'C:\Projects\OGT_S12\OGT Spring 2012 Addresses & Labels 4_24_2012.xlsx' filename_size_selection = 'C:\Projects\OGT_S12\OGT Spring 2012 Addresses & Labels 4_24_2012.xlsx' starttime = time.time() #ac = AsciiWriter(dbcontext = dbcontext,runcontext = runcontext, filename_additional_label = filename_additional_label, filename_size_selection = filename_size_selection) ac = AsciiWriter(dbcontext=dbcontext, runcontext=runcontext) ac.process() endtime = time.time() print 'TOTAL TIME TAKEN = ', endtime - starttime
def _process_keep_columns(self): print '_process_keep_columns' if len(self.keep_columns) <> 0: dbcontext6 = self.rc.getDBContext(cached=False) all_cols_qry = """Select name from sys.columns where object_id = OBJECT_ID('{tablename}')""".format( tablename=self.output_table) all_cols = dbcontext6.execute(all_cols_qry) all_cols = [each[0].encode('ascii').upper() for each in all_cols] drop_cols = [ each for each in all_cols if each not in self.keep_columns ] for each in drop_cols: DROP_QUERY = "ALTER TABLE {tablename} DROP COLUMN {columnname}".format( tablename=self.output_table, columnname=each) print DROP_QUERY dbcontext6.executeNoResults(DROP_QUERY) dbcontext6.close() if __name__ == '__main__': runcontext = SuiteContext('unittest') dbcontext = runcontext.getDBContext(cached=False) sm = SasMerge(dbcontext=dbcontext, runcontext=runcontext, merge_table_1="Employee_merge_1", merge_table_2="Employee_merge_2", mergeids=["Emp_Num"], drop_columns=[], keep_columns=['EMP_AGE'], output_table="Merge_Output_table") sm.process()
def setUp(self): self.run_context = SuiteContext( 'unittest' ) self.db_context = self.run_context.getDBContext( 'unittest' ) self.LOGGER = self.run_context.get_logger()
def setUpClass(cls): cls.runContext = SuiteContext('unittest') cls.db_context = cls.runContext.getDBContext(tag='unittest') clear_all(cls.db_context) cls.testDataDir = os.path.join(cls.runContext.tests_safe_dir, "intake_test")
create_index = "Create Clustered Index {indexname} on {tablename} ({fieldname})".format( indexname='preqc_ix', tablename=self.flat_tablename, fieldname='id') self.db.executeNoResults(create_index) fields = 'flat_table_id, id' for each in self.mc_items_table_names: ixname = each + '_ix' create_index = "CREATE CLUSTERED INDEX {indexname} on {tablename} ({fieldname})".format( indexname=ixname.lower(), tablename=each, fieldname=fields) self.db.executeNoResults(create_index) if __name__ == '__main__': from airassessmentreporting.testutility import SuiteContext RC = SuiteContext('unittest') # RC = RunContext('unittest') dbcontext = RC.getDBContext() print 'dbcontext=', dbcontext x = PreQC( runcontext=RC, dbcontext=dbcontext, # layoutfile='C:\CVS Projects\CSSC Score Reporting\OGT Fall 2012\Intake Layout\OGT_FA12_Op_DataLayout_IntakeLayout.xls', layoutfile= 'C:\CVS Projects\CSSC Score Reporting\OGT Fall 2012\Intake Layout\OGT_FA12_Op_DataLayout_IntakeLayout.xls', # inputfile='C:\SAS\OGT\Input\original-record1.txt', inputfile= 'H:\\share\\Ohio Graduation Tests\\Technical\\2012 October\\ScoreReports\\TextFileFromDRC\\536215_2012OhioOGTFall_Regular.txt', # inputfile='H:/share/Ohio Graduation Tests/Technical/2012 October/ScoreReports/TextFileFromDRC/536215_2012OhioOGTFall_Regular.txt', # inputfile='H:\\share\\Ohio Graduation Tests\\Technical\\2012 July\\ScoreReports\\TextFileFromDRC\\536214_2012OhioOGTSummer_Regular.txt', # inputfile='C:\SAS\OGT\Input\input-1.txt',