def setupAllDB(): """Sets up all databases""" createDatabase(CONFIG_DB['db_name']) runMigrations() setupJobTrackerDB() setupErrorDB() setupUserDB() setupJobQueueDB() setupValidationDB()
def full_database_setup(): """Sets up a clean database based on the model metadata. It also calculates the FK relationships between tables so we can delete them in order. It yields a tuple the _DB and ordered list of tables.""" rand_id = str(randint(1, 9999)) config = dataactcore.config.CONFIG_DB config['db_name'] = 'unittest{}_data_broker'.format(rand_id) dataactcore.config.CONFIG_DB = config createDatabase(config['db_name']) db = dbConnection() runMigrations() creation_order = baseModel.Base.metadata.sorted_tables yield (db, list(reversed(creation_order))) # drop order db.close() dropDatabase(config['db_name'])
def setUpClass(cls): """Set up resources to be shared within a test class""" #TODO: refactor into a pytest class fixtures and inject as necessary # update application's db config options so unittests # run against test databases suite = cls.__name__.lower() config = dataactcore.config.CONFIG_DB cls.num = randint(1, 9999) config['db_name'] = 'unittest{}_{}_data_broker'.format( cls.num, suite) dataactcore.config.CONFIG_DB = config createDatabase(CONFIG_DB['db_name']) runMigrations() app = createApp() app.config['TESTING'] = True app.config['DEBUG'] = False cls.app = TestApp(app) # Allow us to augment default test failure msg w/ more detail cls.longMessage = True # Upload files to S3 (False = skip re-uploading on subsequent runs) cls.uploadFiles = True # Run tests for local broker or not cls.local = CONFIG_BROKER['local'] # This needs to be set to the local directory for error reports if local is True cls.local_file_directory = CONFIG_SERVICES['error_report_path'] # drop and re-create test job db/tables setupJobTrackerDB() # drop and re-create test error db/tables setupErrorDB() # drop and re-create test validation db setupValidationDB() cls.userId = None # constants to use for default submission start and end dates cls.SUBMISSION_START_DEFAULT = datetime(2015, 10, 1) cls.SUBMISSION_END_DEFAULT = datetime(2015, 10, 31)
def setupJobQueueDB(): """Create job tracker tables from model metadata.""" createDatabase(CONFIG_DB['job_queue_db_name'])
def setupUserDB(): """Create user tables from model metadata.""" createDatabase(CONFIG_DB['user_db_name']) runMigrations('user_manager') insertCodes()
def setupErrorDB(): """Create job tracker tables from model metadata.""" createDatabase(CONFIG_DB['error_db_name']) runMigrations('error_data') insertCodes()
def setUpClass(cls): """Set up resources to be shared within a test class""" cls.session_id = "" with createValidatorApp().app_context(): # update application's db config options so unittests # run against test databases suite = cls.__name__.lower() config = dataactcore.config.CONFIG_DB cls.num = randint(1, 9999) config['db_name'] = 'unittest{}_{}_data_broker'.format( cls.num, suite) dataactcore.config.CONFIG_DB = config createDatabase(CONFIG_DB['db_name']) runMigrations() # drop and re-create test user db/tables setupUserDB() # drop and re-create test job db/tables setupJobTrackerDB() # drop and re-create test error db/tables setupErrorDB() # drop and re-create test validation db/tables setupValidationDB() # load e-mail templates setupEmails() # set up default e-mails for tests test_users = {} test_users['admin_email'] = '*****@*****.**' test_users['change_user_email'] = '*****@*****.**' test_users['password_reset_email'] = '*****@*****.**' test_users['inactive_email'] = '*****@*****.**' test_users['password_lock_email'] = '*****@*****.**' test_users['expired_lock_email'] = '*****@*****.**' test_users['agency_admin_email'] = '*****@*****.**' # this email is for a regular agency_user email that is to be used for # testing functionality expected by a normal, base user test_users['agency_user'] = '******' test_users['approved_email'] = '*****@*****.**' test_users['submission_email'] = '*****@*****.**' user_password = '******' admin_password = '******' # set up users for status tests StatusTestUser = namedtuple( 'StatusTestUser', ['email', 'user_status', 'permissions', 'user_type']) StatusTestUser.__new__.__defaults__ = (None, None, AccountType.AGENCY_USER, None) status_test_users = [] status_test_users.append( StatusTestUser('*****@*****.**', 'awaiting_confirmation', 0)) status_test_users.append( StatusTestUser('*****@*****.**', 'email_confirmed')) status_test_users.append( StatusTestUser('*****@*****.**', 'awaiting_approval')) status_test_users.append( StatusTestUser('*****@*****.**', 'awaiting_approval')) status_test_users.append( StatusTestUser('*****@*****.**', 'awaiting_approval')) status_test_users.append( StatusTestUser( test_users['admin_email'], 'approved', AccountType.WEBSITE_ADMIN + AccountType.AGENCY_USER)) status_test_users.append( StatusTestUser(test_users['approved_email'], 'approved')) status_test_users.append( StatusTestUser('*****@*****.**', 'denied')) # add new users createUserWithPassword(test_users["submission_email"], user_password, Bcrypt()) createUserWithPassword(test_users["change_user_email"], user_password, Bcrypt()) createUserWithPassword(test_users["password_reset_email"], user_password, Bcrypt()) createUserWithPassword(test_users["inactive_email"], user_password, Bcrypt()) createUserWithPassword(test_users["password_lock_email"], user_password, Bcrypt()) createUserWithPassword(test_users['expired_lock_email'], user_password, Bcrypt()) createUserWithPassword(test_users['agency_admin_email'], admin_password, Bcrypt(), permission=4) createUserWithPassword(test_users['agency_user'], user_password, Bcrypt()) # get user info and save as class variables for use by tests sess = GlobalDB.db().session agencyUser = sess.query(User).filter( User.email == test_users['agency_user']).one() cls.agency_user_id = agencyUser.user_id # set the specified account to be expired expiredUser = sess.query(User).filter( User.email == test_users['expired_lock_email']).one() today = parse(time.strftime("%c")) expiredUser.last_login_date = (today - timedelta(days=120)).strftime("%c") sess.add(expiredUser) # create users for status testing for u in status_test_users: user = User(email=u.email, permissions=u.permissions, user_status=sess.query(UserStatus).filter( UserStatus.name == u.user_status).one()) sess.add(user) # set up approved user user = sess.query(User).filter( User.email == test_users['approved_email']).one() user.username = "******" user.cgac_code = "000" user.salt, user.password_hash = getPasswordHash( user_password, Bcrypt()) sess.add(user) cls.approved_user_id = user.user_id # set up admin user admin = sess.query(User).filter( User.email == test_users['admin_email']).one() admin.salt, admin.password_hash = getPasswordHash( admin_password, Bcrypt()) admin.name = "Mr. Manager" admin.cgac_code = "SYS" sess.add(admin) # set up status changed user statusChangedUser = sess.query(User).filter( User.email == test_users['change_user_email']).one() statusChangedUser.name = "Test User" statusChangedUser.user_status = sess.query(UserStatus).filter( UserStatus.name == 'email_confirmed').one() sess.add(statusChangedUser) cls.status_change_user_id = statusChangedUser.user_id # set up deactivated user deactivated_user = sess.query(User).filter( User.email == test_users['inactive_email']).one() deactivated_user.last_login_date = time.strftime("%c") deactivated_user.is_active = False sess.add(deactivated_user) sess.commit() # get lookup dictionaries cls.jobStatusDict = lookups.JOB_STATUS_DICT cls.jobTypeDict = lookups.JOB_TYPE_DICT cls.fileTypeDict = lookups.FILE_TYPE_DICT cls.fileStatusDict = lookups.FILE_STATUS_DICT cls.ruleSeverityDict = lookups.RULE_SEVERITY_DICT cls.errorTypeDict = lookups.ERROR_TYPE_DICT cls.publishStatusDict = lookups.PUBLISH_STATUS_DICT cls.userStatusDict = lookups.USER_STATUS_DICT # set up info needed by the individual test classes cls.test_users = test_users cls.user_password = user_password cls.admin_password = admin_password cls.local = CONFIG_BROKER['local']
def setupValidationDB(): """Create validation tables from model metadata and do initial inserts.""" createDatabase(CONFIG_DB['validator_db_name']) runMigrations('validation') insertCodes()
def setupValidationDB(hardReset = False): """Create validation tables from model metadata and do initial inserts.""" createDatabase(CONFIG_DB['validator_db_name']) validatorDb = ValidatorValidationInterface() # TODO: use Alembic for initial db setup if hardReset: validationModels.Base.metadata.drop_all(validatorDb.engine) validationModels.Base.metadata.create_all(validatorDb.engine) validatorDb.session.commit() validatorDb.session.close() # insert rule timing ruleTimingList = [(1,'file_validation','Run during pre-load validation of a file'), (2,'prerequisite','Run only when referenced by another rule'), (3,'cross-file','This rule is checked during cross-file validation')] for r in ruleTimingList: ruleTiming = RuleTiming(rule_timing_id = r[0], name = r[1], description = r[2]) validatorDb.session.merge(ruleTiming) # insert file types fileTypeList = [(1, 'award', 'award file'), (2, 'award_financial', 'award_financial file'), (3, 'appropriations', 'appropriations file'), (4, 'program_activity','program activity and object class file')] for f in fileTypeList: fileType = FileType(file_id=f[0], name=f[1], description=f[2]) validatorDb.session.merge(fileType) # insert rule types ruleTypeList = [(1, 'TYPE', 'checks type'), (2, 'EQUAL', 'equals operatior '), (3, 'NOT EQUAL', 'not equals operator '), (4, 'LESS', 'less than operator '), (5, 'GREATER', 'greater than operator'), (6, 'LENGTH', 'string length'), (7, 'IN_SET', 'value must be in set'), (8, 'MIN LENGTH', 'length of data must be at least reference value'), (9, 'REQUIRED_CONDITIONAL', 'field is required if secondary rule passes'), (10, 'SUM', 'field is equal to the sum of other fields') ] for r in ruleTypeList: ruleType = RuleType(rule_type_id=r[0], name=r[1], description=r[2]) validatorDb.session.merge(ruleType) # insert field types fieldTypeList = [(1, 'INT', 'integer type'), (2, 'DECIMAL', 'decimal type '), (3, 'BOOLEAN', 'yes/no'), (4, 'STRING', 'string type'), (5, 'LONG', 'long integer')] for f in fieldTypeList: fieldType = FieldType(field_type_id=f[0], name=f[1], description=f[2]) validatorDb.session.merge(fieldType) # insert multi-field rule types mfrTypeList = [(1, 'CAR_MATCH', 'Matching a set of fields against a CAR file'), (2, 'FIELD_MATCH', 'Match a set of fields against a different file'), (3, 'RULE_IF', 'Apply first rule if second rule passes'), (4, 'GREATER', 'Check if field is greater than specified value'), (5, 'SUM_TO_VALUE', 'Sum a set of fields and compare to specified value') ] for m in mfrTypeList: mfrt = MultiFieldRuleType( multi_field_rule_type_id = m[0], name=m[1], description=m[2]) validatorDb.session.merge(mfrt) validatorDb.session.commit() validatorDb.session.close()
def setupJobTrackerDB(): """Create job tracker tables from model metadata.""" createDatabase(CONFIG_DB['job_db_name']) runMigrations('job_tracker') insertCodes()
def setupStagingDB(): """Create the staging database.""" createDatabase(CONFIG_DB['staging_db_name'])
def setupStagingDB(): """Create job tracker tables from model metadata.""" createDatabase(CONFIG_DB['staging_db_name']) runMigrations('staging')
def setupValidationDB(hardReset=False): """Create validation tables from model metadata and do initial inserts.""" createDatabase(CONFIG_DB['validator_db_name']) validatorDb = ValidatorValidationInterface() # TODO: use Alembic for initial db setup if hardReset: validationModels.Base.metadata.drop_all(validatorDb.engine) validationModels.Base.metadata.create_all(validatorDb.engine) validatorDb.session.commit() validatorDb.session.close() # insert rule timing ruleTimingList = [ (1, 'file_validation', 'Run during pre-load validation of a file'), (2, 'prerequisite', 'Run only when referenced by another rule'), (3, 'cross-file', 'This rule is checked during cross-file validation') ] for r in ruleTimingList: ruleTiming = RuleTiming(rule_timing_id=r[0], name=r[1], description=r[2]) validatorDb.session.merge(ruleTiming) # insert file types fileTypeList = [(1, 'award', 'award file'), (2, 'award_financial', 'award_financial file'), (3, 'appropriations', 'appropriations file'), (4, 'program_activity', 'program activity and object class file')] for f in fileTypeList: fileType = FileType(file_id=f[0], name=f[1], description=f[2]) validatorDb.session.merge(fileType) # insert rule types ruleTypeList = [(1, 'TYPE', 'checks type'), (2, 'EQUAL', 'equals operatior '), (3, 'NOT EQUAL', 'not equals operator '), (4, 'LESS', 'less than operator '), (5, 'GREATER', 'greater than operator'), (6, 'LENGTH', 'string length'), (7, 'IN_SET', 'value must be in set'), (8, 'MIN LENGTH', 'length of data must be at least reference value'), (9, 'REQUIRED_CONDITIONAL', 'field is required if secondary rule passes'), (10, 'SUM', 'field is equal to the sum of other fields')] for r in ruleTypeList: ruleType = RuleType(rule_type_id=r[0], name=r[1], description=r[2]) validatorDb.session.merge(ruleType) # insert field types fieldTypeList = [(1, 'INT', 'integer type'), (2, 'DECIMAL', 'decimal type '), (3, 'BOOLEAN', 'yes/no'), (4, 'STRING', 'string type'), (5, 'LONG', 'long integer')] for f in fieldTypeList: fieldType = FieldType(field_type_id=f[0], name=f[1], description=f[2]) validatorDb.session.merge(fieldType) # insert multi-field rule types mfrTypeList = [ (1, 'CAR_MATCH', 'Matching a set of fields against a CAR file'), (2, 'FIELD_MATCH', 'Match a set of fields against a different file'), (3, 'RULE_IF', 'Apply first rule if second rule passes'), (4, 'GREATER', 'Check if field is greater than specified value'), (5, 'SUM_TO_VALUE', 'Sum a set of fields and compare to specified value') ] for m in mfrTypeList: mfrt = MultiFieldRuleType(multi_field_rule_type_id=m[0], name=m[1], description=m[2]) validatorDb.session.merge(mfrt) validatorDb.session.commit() validatorDb.session.close()