def load_definitions(sess, force_tas_load, ruleList=None):
"""Load file definitions."""
validator_config_path = os.path.join(CONFIG_BROKER["path"],
"dataactvalidator", "config")
integration_test_data_path = os.path.join(CONFIG_BROKER["path"],
"tests", "integration",
"data")
SchemaLoader.loadAllFromPath(validator_config_path)
SQLLoader.loadSql("sqlRules.csv")
if ruleList is not None:
# If rule list provided, drop all other rules
sess.query(RuleSql).filter(not_(
RuleSql.rule_label.in_(ruleList))).delete(
synchronize_session='fetch')
sess.commit()
# Load domain values tables
loadDomainValues(
validator_config_path,
os.path.join(integration_test_data_path, "program_activity.csv"))
if sess.query(TASLookup).count() == 0 or force_tas_load:
# TAS table is empty, load it
loadTas(tasFile=os.path.join(integration_test_data_path,
"cars_tas.csv"))
# Load test SF-133
load_all_sf133(integration_test_data_path)
def error_rows(rule_file,
staging_db,
submission=None,
models=None,
assert_num=None):
"""Insert the models into the database, then run the rule SQL against
those models. Return the resulting (invalid) rows"""
if submission is None:
submission = Submission(user_id=None,
reporting_start_date=datetime(2015, 10, 1),
reporting_end_date=datetime(2015, 10, 31))
if models is None:
models = []
submission_id = insert_submission(staging_db, submission)
sql = SQLLoader.readSqlStr(rule_file).format(submission_id)
for model in models:
model.submission_id = submission_id
staging_db.session.add(model)
staging_db.session.commit()
result = staging_db.connection.execute(sql).fetchall()
if assert_num is not None:
assert (len(result) == assert_num)
# clean up
for model in models:
staging_db.session.delete(model)
staging_db.session.commit()
return result
def error_rows(rule_file, staging_db, submission=None, models=None, assert_num=None):
"""Insert the models into the database, then run the rule SQL against
those models. Return the resulting (invalid) rows"""
if submission is None:
submission = Submission(
user_id=None, reporting_start_date=datetime(2015, 10, 1),
reporting_end_date=datetime(2015, 10, 31))
if models is None:
models = []
submission_id = insert_submission(staging_db, submission)
sql = SQLLoader.read_sql_str(rule_file).format(submission_id)
for model in models:
model.submission_id = submission_id
staging_db.session.add(model)
staging_db.session.commit()
result = staging_db.connection.execute(sql).fetchall()
if assert_num is not None:
assert(len(result) == assert_num)
return result
def load_sql_rules():
"""Load the SQL-based validation rules."""
logger.info('Loading SQL-based validation rules')
SQLLoader.load_sql("sqlRules.csv")
def query_columns(rule_file, staging_db):
sql = SQLLoader.read_sql_str(rule_file).format(randint(1, 9999))
return staging_db.connection.execute(sql).keys()
def load_sql_rules():
"""Load the SQL-based validation rules."""
logger.info('Loading SQL-based validation rules')
SQLLoader.load_sql("sqlRules.csv")
def load_sql_rules():
"""Load the SQL-based validation rules."""
logger.info('Loading SQL-based validation rules')
SQLLoader.load_sql("sqlRules.csv")
logger.info('Loading non-SQL-based validation labels')
LabelLoader.load_labels("validationLabels.csv")
def load_sql_rules():
"""Load the SQL-based validation rules."""
logger.info('Loading SQL-based validation rules')
SQLLoader.load_sql("sqlRules.csv")
logger.info('Loading non-SQL-based validation labels')
LabelLoader.load_labels("validationLabels.csv")
