def load_definitions(sess, force_tas_load, ruleList=None): """Load file definitions.""" validator_config_path = os.path.join(CONFIG_BROKER["path"], "dataactvalidator", "config") integration_test_data_path = os.path.join(CONFIG_BROKER["path"], "tests", "integration", "data") SchemaLoader.loadAllFromPath(validator_config_path) SQLLoader.loadSql("sqlRules.csv") if ruleList is not None: # If rule list provided, drop all other rules sess.query(RuleSql).filter(not_( RuleSql.rule_label.in_(ruleList))).delete( synchronize_session='fetch') sess.commit() # Load domain values tables loadDomainValues( validator_config_path, os.path.join(integration_test_data_path, "program_activity.csv")) if sess.query(TASLookup).count() == 0 or force_tas_load: # TAS table is empty, load it loadTas(tasFile=os.path.join(integration_test_data_path, "cars_tas.csv")) # Load test SF-133 load_all_sf133(integration_test_data_path)
def error_rows(rule_file, staging_db, submission=None, models=None, assert_num=None): """Insert the models into the database, then run the rule SQL against those models. Return the resulting (invalid) rows""" if submission is None: submission = Submission(user_id=None, reporting_start_date=datetime(2015, 10, 1), reporting_end_date=datetime(2015, 10, 31)) if models is None: models = [] submission_id = insert_submission(staging_db, submission) sql = SQLLoader.readSqlStr(rule_file).format(submission_id) for model in models: model.submission_id = submission_id staging_db.session.add(model) staging_db.session.commit() result = staging_db.connection.execute(sql).fetchall() if assert_num is not None: assert (len(result) == assert_num) # clean up for model in models: staging_db.session.delete(model) staging_db.session.commit() return result
def error_rows(rule_file, staging_db, submission=None, models=None, assert_num=None): """Insert the models into the database, then run the rule SQL against those models. Return the resulting (invalid) rows""" if submission is None: submission = Submission( user_id=None, reporting_start_date=datetime(2015, 10, 1), reporting_end_date=datetime(2015, 10, 31)) if models is None: models = [] submission_id = insert_submission(staging_db, submission) sql = SQLLoader.read_sql_str(rule_file).format(submission_id) for model in models: model.submission_id = submission_id staging_db.session.add(model) staging_db.session.commit() result = staging_db.connection.execute(sql).fetchall() if assert_num is not None: assert(len(result) == assert_num) return result
def load_sql_rules(): """Load the SQL-based validation rules.""" logger.info('Loading SQL-based validation rules') SQLLoader.load_sql("sqlRules.csv")
def query_columns(rule_file, staging_db): sql = SQLLoader.read_sql_str(rule_file).format(randint(1, 9999)) return staging_db.connection.execute(sql).keys()
def load_sql_rules(): """Load the SQL-based validation rules.""" logger.info('Loading SQL-based validation rules') SQLLoader.load_sql("sqlRules.csv") logger.info('Loading non-SQL-based validation labels') LabelLoader.load_labels("validationLabels.csv")