def load_definitions(sess, force_tas_load, ruleList=None):
        """Load file definitions."""
        validator_config_path = os.path.join(CONFIG_BROKER["path"],
                                             "dataactvalidator", "config")
        integration_test_data_path = os.path.join(CONFIG_BROKER["path"],
                                                  "tests", "integration",
                                                  "data")

        SchemaLoader.loadAllFromPath(validator_config_path)
        SQLLoader.loadSql("sqlRules.csv")

        if ruleList is not None:
            # If rule list provided, drop all other rules
            sess.query(RuleSql).filter(not_(
                RuleSql.rule_label.in_(ruleList))).delete(
                    synchronize_session='fetch')
            sess.commit()

        # Load domain values tables
        loadDomainValues(
            validator_config_path,
            os.path.join(integration_test_data_path, "program_activity.csv"))
        if sess.query(TASLookup).count() == 0 or force_tas_load:
            # TAS table is empty, load it
            loadTas(tasFile=os.path.join(integration_test_data_path,
                                         "cars_tas.csv"))

        # Load test SF-133
        load_all_sf133(integration_test_data_path)
def loadValidator():
    """Load validator fields and rules from config."""
    validator_config_path = os.path.join(
    os.path.dirname(dataactvalidator.__file__), "config")
    appropriationsFields = os.path.join(validator_config_path, "appropFields.csv")
    try:
        SchemaLoader.loadFields("appropriations", appropriationsFields)
    except IOError:
        print("Can't open file: {}".format(appropriationsFields))


    appropriationsRules = os.path.join(validator_config_path, "appropRules.csv")
    try:
        SchemaLoader.loadRules("appropriations", appropriationsRules)
    except IOError:
        print("Can't open file: {}".format(appropriationsRules))
Esempio n. 3
0
def loadValidator():
    """Load validator fields and rules from config."""
    validator_config_path = os.path.join(
        os.path.dirname(dataactvalidator.__file__), "config")
    appropriationsFields = os.path.join(validator_config_path,
                                        "appropFields.csv")
    try:
        SchemaLoader.loadFields("appropriations", appropriationsFields)
    except IOError:
        print("Can't open file: {}".format(appropriationsFields))

    appropriationsRules = os.path.join(validator_config_path,
                                       "appropRules.csv")
    try:
        SchemaLoader.loadRules("appropriations", appropriationsRules)
    except IOError:
        print("Can't open file: {}".format(appropriationsRules))
def load_validator_schema():
    """Load file-level .csv schemas into the broker database."""
    logger.info('Loading validator schemas')
    SchemaLoader.load_all_from_path(validator_config_path)
def load_validator_schema():
    """Load file-level .csv schemas into the broker database."""
    logger.info('Loading validator schemas')
    SchemaLoader.load_all_from_path(validator_config_path)
Esempio n. 6
0
 def load_definitions(interfaces, force_tas_load):
     """Load file definitions."""
     # TODO: introduce flexibility re: test file location
     SchemaLoader.loadFields(
         "appropriations",
         join(CONFIG_BROKER["path"], "dataactvalidator", "config",
              "appropFields.csv"))
     SchemaLoader.loadFields(
         "program_activity",
         join(CONFIG_BROKER["path"], "dataactvalidator", "config",
              "programActivityFields.csv"))
     SchemaLoader.loadFields(
         "award_financial",
         join(CONFIG_BROKER["path"], "dataactvalidator", "config",
              "awardFinancialFields.csv"))
     SchemaLoader.loadFields(
         "award",
         join(CONFIG_BROKER["path"], "dataactvalidator", "config",
              "awardFields.csv"))
     SchemaLoader.loadRules(
         "appropriations",
         join(CONFIG_BROKER["path"], "dataactvalidator", "config",
              "appropRules.csv"))
     SchemaLoader.loadRules(
         "program_activity",
         join(CONFIG_BROKER["path"], "dataactvalidator", "config",
              "programActivityRules.csv"))
     SchemaLoader.loadRules(
         "award_financial",
         join(CONFIG_BROKER["path"], "dataactvalidator", "config",
              "awardFinancialRules.csv"))
     SchemaLoader.loadCrossRules(
         join(CONFIG_BROKER["path"], "dataactvalidator", "config",
              "crossFileRules.csv"))
     # Load domain values tables
     loadDomainValues("../dataactvalidator/config/")
     if (interfaces.validationDb.session.query(TASLookup).count() == 0
             or force_tas_load):
         # TAS table is empty, load it
         loadTas(tasFile="all_tas_betc.csv", dropIdx=False)
""" This script loads fields and rules for the appropriations file type """
from dataactvalidator.filestreaming.schemaLoader import SchemaLoader

SchemaLoader.loadFields("appropriations", "../tests/appropriationsFields.csv")
SchemaLoader.loadRules("appropriations", "../tests/appropriationsRules.csv")
Esempio n. 8
0
    def setUpClass(cls):
        """Set up resources to be shared within a test class"""
        # TODO: refactor into a pytest class fixtures and inject as necessary
        # update application's db config options so unittests
        # run against test databases
        configure_logging()
        suite = cls.__name__.lower()
        config = dataactcore.config.CONFIG_DB
        cls.num = randint(1, 9999)
        config['db_name'] = 'unittest{}_{}_data_broker'.format(cls.num, suite)
        dataactcore.config.CONFIG_DB = config
        create_database(CONFIG_DB['db_name'])
        run_migrations()

        app = create_app()
        app.config['TESTING'] = True
        app.config['DEBUG'] = False
        cls.app = TestApp(app)
        sess = GlobalDB.db().session

        # set up default e-mails for tests
        test_users = {
            'admin_user': '******',
            'agency_user': '******',
            'agency_user_2': '*****@*****.**',
            'no_permissions_user': '******',
            'editfabs_user': '******'
        }
        admin_password = '******'

        cgac = CGAC(cgac_code='000', agency_name='Example Agency')
        sess.add(cgac)
        sess.commit()

        # Allow us to augment default test failure msg w/ more detail
        cls.longMessage = True
        # Upload files to S3 (False = skip re-uploading on subsequent runs)
        cls.uploadFiles = True
        # Run tests for local broker or not
        cls.local = CONFIG_BROKER['local']
        # This needs to be set to the local directory for error reports if local is True
        cls.local_file_directory = CONFIG_SERVICES['error_report_path']

        # drop and re-create test job db/tables
        setup_job_tracker_db()
        # drop and re-create test error db/tables
        setup_error_db()
        # drop and re-create test validation db
        setup_validation_db()

        # setup Schema

        SchemaLoader.load_all_from_path(validator_config_path)
        load_sql_rules()

        create_user_with_password(test_users["admin_user"],
                                  admin_password,
                                  Bcrypt(),
                                  website_admin=True)
        cls.userId = None
        cls.test_users = test_users
        # constants to use for default submission start and end dates
        cls.SUBMISSION_START_DEFAULT = datetime(2015, 10, 1)
        cls.SUBMISSION_END_DEFAULT = datetime(2015, 10, 31)
 def load_definitions(interfaces, force_tas_load):
     """Load file definitions."""
     # TODO: introduce flexibility re: test file location
     SchemaLoader.loadFields("appropriations","../dataactvalidator/config/appropFields.csv")
     SchemaLoader.loadFields("program_activity","../dataactvalidator/config/programActivityFields.csv")
     SchemaLoader.loadFields("award_financial","../dataactvalidator/config/awardFinancialFields.csv")
     SchemaLoader.loadFields("award","../dataactvalidator/config/awardFields.csv")
     SchemaLoader.loadRules("appropriations","../dataactvalidator/config/appropRules.csv")
     SchemaLoader.loadRules("program_activity","../dataactvalidator/config/programActivityRules.csv")
     SchemaLoader.loadRules("award_financial","../dataactvalidator/config/awardFinancialRules.csv")
     SchemaLoader.loadCrossRules("../dataactvalidator/config/crossFileRules.csv")
     if (interfaces.validationDb.session.query(TASLookup).count() == 0
             or force_tas_load):
         # TAS table is empty, load it
         loadTas(tasFile="all_tas_betc.csv", dropIdx=False)
Esempio n. 10
0
def loadValidator():
    """Load validator fields and rules from config."""
    basePath = CONFIG_BROKER["path"]
    validator_config_path = os.path.join(basePath, "dataactvalidator", "config")
    SchemaLoader.loadAllFromPath(validator_config_path)
Esempio n. 11
0
def loadValidator():
    """Load validator fields and rules from config."""
    basePath = CONFIG_BROKER["path"]
    validator_config_path = os.path.join(basePath, "dataactvalidator",
                                         "config")
    SchemaLoader.loadAllFromPath(validator_config_path)
""" This script loads fields and rules for the appropriations file type """
from dataactvalidator.filestreaming.schemaLoader import SchemaLoader

SchemaLoader.loadFields("appropriations","../tests/appropriationsFields.csv")
SchemaLoader.loadRules("appropriations","../tests/appropriationsRules.csv")