def load_definitions(sess, force_tas_load, ruleList=None): """Load file definitions.""" validator_config_path = os.path.join(CONFIG_BROKER["path"], "dataactvalidator", "config") integration_test_data_path = os.path.join(CONFIG_BROKER["path"], "tests", "integration", "data") SchemaLoader.loadAllFromPath(validator_config_path) SQLLoader.loadSql("sqlRules.csv") if ruleList is not None: # If rule list provided, drop all other rules sess.query(RuleSql).filter(not_( RuleSql.rule_label.in_(ruleList))).delete( synchronize_session='fetch') sess.commit() # Load domain values tables loadDomainValues( validator_config_path, os.path.join(integration_test_data_path, "program_activity.csv")) if sess.query(TASLookup).count() == 0 or force_tas_load: # TAS table is empty, load it loadTas(tasFile=os.path.join(integration_test_data_path, "cars_tas.csv")) # Load test SF-133 load_all_sf133(integration_test_data_path)
def loadValidator(): """Load validator fields and rules from config.""" validator_config_path = os.path.join( os.path.dirname(dataactvalidator.__file__), "config") appropriationsFields = os.path.join(validator_config_path, "appropFields.csv") try: SchemaLoader.loadFields("appropriations", appropriationsFields) except IOError: print("Can't open file: {}".format(appropriationsFields)) appropriationsRules = os.path.join(validator_config_path, "appropRules.csv") try: SchemaLoader.loadRules("appropriations", appropriationsRules) except IOError: print("Can't open file: {}".format(appropriationsRules))
def load_validator_schema(): """Load file-level .csv schemas into the broker database.""" logger.info('Loading validator schemas') SchemaLoader.load_all_from_path(validator_config_path)
def load_definitions(interfaces, force_tas_load): """Load file definitions.""" # TODO: introduce flexibility re: test file location SchemaLoader.loadFields( "appropriations", join(CONFIG_BROKER["path"], "dataactvalidator", "config", "appropFields.csv")) SchemaLoader.loadFields( "program_activity", join(CONFIG_BROKER["path"], "dataactvalidator", "config", "programActivityFields.csv")) SchemaLoader.loadFields( "award_financial", join(CONFIG_BROKER["path"], "dataactvalidator", "config", "awardFinancialFields.csv")) SchemaLoader.loadFields( "award", join(CONFIG_BROKER["path"], "dataactvalidator", "config", "awardFields.csv")) SchemaLoader.loadRules( "appropriations", join(CONFIG_BROKER["path"], "dataactvalidator", "config", "appropRules.csv")) SchemaLoader.loadRules( "program_activity", join(CONFIG_BROKER["path"], "dataactvalidator", "config", "programActivityRules.csv")) SchemaLoader.loadRules( "award_financial", join(CONFIG_BROKER["path"], "dataactvalidator", "config", "awardFinancialRules.csv")) SchemaLoader.loadCrossRules( join(CONFIG_BROKER["path"], "dataactvalidator", "config", "crossFileRules.csv")) # Load domain values tables loadDomainValues("../dataactvalidator/config/") if (interfaces.validationDb.session.query(TASLookup).count() == 0 or force_tas_load): # TAS table is empty, load it loadTas(tasFile="all_tas_betc.csv", dropIdx=False)
""" This script loads fields and rules for the appropriations file type """ from dataactvalidator.filestreaming.schemaLoader import SchemaLoader SchemaLoader.loadFields("appropriations", "../tests/appropriationsFields.csv") SchemaLoader.loadRules("appropriations", "../tests/appropriationsRules.csv")
def setUpClass(cls): """Set up resources to be shared within a test class""" # TODO: refactor into a pytest class fixtures and inject as necessary # update application's db config options so unittests # run against test databases configure_logging() suite = cls.__name__.lower() config = dataactcore.config.CONFIG_DB cls.num = randint(1, 9999) config['db_name'] = 'unittest{}_{}_data_broker'.format(cls.num, suite) dataactcore.config.CONFIG_DB = config create_database(CONFIG_DB['db_name']) run_migrations() app = create_app() app.config['TESTING'] = True app.config['DEBUG'] = False cls.app = TestApp(app) sess = GlobalDB.db().session # set up default e-mails for tests test_users = { 'admin_user': '******', 'agency_user': '******', 'agency_user_2': '*****@*****.**', 'no_permissions_user': '******', 'editfabs_user': '******' } admin_password = '******' cgac = CGAC(cgac_code='000', agency_name='Example Agency') sess.add(cgac) sess.commit() # Allow us to augment default test failure msg w/ more detail cls.longMessage = True # Upload files to S3 (False = skip re-uploading on subsequent runs) cls.uploadFiles = True # Run tests for local broker or not cls.local = CONFIG_BROKER['local'] # This needs to be set to the local directory for error reports if local is True cls.local_file_directory = CONFIG_SERVICES['error_report_path'] # drop and re-create test job db/tables setup_job_tracker_db() # drop and re-create test error db/tables setup_error_db() # drop and re-create test validation db setup_validation_db() # setup Schema SchemaLoader.load_all_from_path(validator_config_path) load_sql_rules() create_user_with_password(test_users["admin_user"], admin_password, Bcrypt(), website_admin=True) cls.userId = None cls.test_users = test_users # constants to use for default submission start and end dates cls.SUBMISSION_START_DEFAULT = datetime(2015, 10, 1) cls.SUBMISSION_END_DEFAULT = datetime(2015, 10, 31)
def load_definitions(interfaces, force_tas_load): """Load file definitions.""" # TODO: introduce flexibility re: test file location SchemaLoader.loadFields("appropriations","../dataactvalidator/config/appropFields.csv") SchemaLoader.loadFields("program_activity","../dataactvalidator/config/programActivityFields.csv") SchemaLoader.loadFields("award_financial","../dataactvalidator/config/awardFinancialFields.csv") SchemaLoader.loadFields("award","../dataactvalidator/config/awardFields.csv") SchemaLoader.loadRules("appropriations","../dataactvalidator/config/appropRules.csv") SchemaLoader.loadRules("program_activity","../dataactvalidator/config/programActivityRules.csv") SchemaLoader.loadRules("award_financial","../dataactvalidator/config/awardFinancialRules.csv") SchemaLoader.loadCrossRules("../dataactvalidator/config/crossFileRules.csv") if (interfaces.validationDb.session.query(TASLookup).count() == 0 or force_tas_load): # TAS table is empty, load it loadTas(tasFile="all_tas_betc.csv", dropIdx=False)
def loadValidator(): """Load validator fields and rules from config.""" basePath = CONFIG_BROKER["path"] validator_config_path = os.path.join(basePath, "dataactvalidator", "config") SchemaLoader.loadAllFromPath(validator_config_path)
""" This script loads fields and rules for the appropriations file type """ from dataactvalidator.filestreaming.schemaLoader import SchemaLoader SchemaLoader.loadFields("appropriations","../tests/appropriationsFields.csv") SchemaLoader.loadRules("appropriations","../tests/appropriationsRules.csv")