def main():
'''
Main Entry for ad-hoc testing to trigger batcher
'''
parser = ArgumentParser(description='File Batcher entry point')
parser.add_argument(
'-p',
dest='pid_file',
default='/opt/edware/run/file-monitor-smarter_score_batcher.pid',
help="pid file for TSB file monitor daemon")
parser.add_argument('-d',
dest='daemon',
action='store_true',
default=False,
help="daemon mode for TSB file monitor")
parser.add_argument('-i',
dest='ini_file',
default='/opt/edware/conf/smarter_score_batcher.ini',
help="ini file")
args = parser.parse_args()
if args.daemon:
create_daemon(args.pid_file)
file = args.ini_file
logging.config.fileConfig(file)
settings = read_ini(file)
initialize_db(TSBDBConnection, settings)
if args.daemon:
run_cron_sync_file(settings)
while True:
time.sleep(1)
else:
move_to_staging(settings)
def test_parse_db_settings_with_multi_tenancy(self):
settings = {'edware.db.echo': 'False',
'edware.db.dummyTenant.echo': 'True',
'edware.db.schema_name': 'myname',
'edware.db.dummyTenant.schema_name': 'dummySchema',
'edware.db.dummyTenant.url': 'sqlite:///:memory:',
'edware.db.dummyTenant.state_code': 'NC',
'edware.db.aTenant.url': 'sqlite:///:memory:',
'edware.db.aTenant.state_code': 'AB',
'edware.db.bTenant.url': 'sqlite:///:memory:',
'edware.db.bTenant.state_code': 'AB',
'edware.db.bTenant.echo': 'True',
'ignoreMe': 'setting',
'dummy': 'other settings'}
initialize_db(EdCoreDBConnection, settings)
self.assertEquals(len(get_data_source_names()), 3)
self.assertIn('edware.db.dummyTenant', get_data_source_names())
self.assertIn('edware.db.aTenant', get_data_source_names())
self.assertIn('edware.db.bTenant', get_data_source_names())
dbUtil = component.queryUtility(IDbUtil, 'edware.db.dummyTenant')
self.assertIsNotNone(dbUtil)
self.assertEqual(dbUtil.get_engine().echo, True)
self.assertEqual(dbUtil.get_metadata().schema, settings['edware.db.dummyTenant.schema_name'])
self.assertEqual(dbUtil.get_engine().url.database, ':memory:')
dbUtil = component.queryUtility(IDbUtil, 'edware.db.aTenant')
self.assertIsNotNone(dbUtil)
self.assertEqual(dbUtil.get_engine().echo, False)
self.assertEqual(dbUtil.get_metadata().schema, settings['edware.db.schema_name'])
self.assertEqual(dbUtil.get_engine().url.database, ':memory:')
dbUtil = component.queryUtility(IDbUtil, 'edware.db.bTenant')
self.assertIsNotNone(dbUtil)
self.assertEqual(dbUtil.get_engine().echo, True)
self.assertEqual(dbUtil.get_metadata().schema, settings['edware.db.schema_name'])
self.assertEqual(dbUtil.get_engine().url.database, ':memory:')
def initialize_all_db(udl2_conf, udl2_flat_conf):
# init db engine
initialize_db_prod(udl2_conf)
initialize_db_target(udl2_conf)
initialize_db_udl(udl2_conf)
# using edcore connection class to init statsdb connection
# this needs a flat config file rather than udl2 which needs nested config
edcoredb.initialize_db(StatsDBConnection, udl2_flat_conf, allow_schema_create=True)
def test_initialize_db_without_tenants(self):
settings = {'edware_stats.db.schema_name': 'dummySchema',
'edware_stats.db.url': 'sqlite:///:memory:'}
initialize_db(StatsDBConnection, settings)
self.assertEquals(len(get_data_source_names()), 1)
self.assertIn('edware_stats.db', get_data_source_names())
dbUtil = component.queryUtility(IDbUtil, 'edware_stats.db')
self.assertIsNotNone(dbUtil)
self.assertEqual(dbUtil.get_metadata().schema, settings['edware_stats.db.schema_name'])
def test_initialize_db_without_tenants(self):
settings = {
'edware_stats.db.schema_name': 'dummySchema',
'edware_stats.db.url': 'sqlite:///:memory:'
}
initialize_db(StatsDBConnection, settings)
self.assertEquals(len(get_data_source_names()), 1)
self.assertIn('edware_stats.db', get_data_source_names())
dbUtil = component.queryUtility(IDbUtil, 'edware_stats.db')
self.assertIsNotNone(dbUtil)
self.assertEqual(dbUtil.get_metadata().schema,
settings['edware_stats.db.schema_name'])
def test_parse_db_settings_with_no_generic_settings(self):
settings = {'edware.db.dummyTenant.echo': 'True',
'edware.db.dummyTenant.schema_name': 'dummySchema',
'edware.db.dummyTenant.url': 'sqlite:///:memory:',
'edware.db.dummyTenant.state_code': 'NC',
'ignoreMe': 'setting',
'dummy': 'other settings'}
initialize_db(EdCoreDBConnection, settings)
dbUtil = component.queryUtility(IDbUtil, 'edware.db.dummyTenant')
self.assertIsNotNone(dbUtil)
self.assertEqual(dbUtil.get_engine().echo, True)
self.assertEqual(dbUtil.get_metadata().schema, settings['edware.db.dummyTenant.schema_name'])
self.assertEqual(dbUtil.get_engine().url.database, ':memory:')
def setup_celery(settings, prefix=PREFIX, db_connection=True):
'''
Setup celery based on parameters defined in setting (ini file).
This calls by client application when dictionary of settings is given
:param settings: dict of configurations
:param prefix: prefix in configurations used for configuring celery
'''
global conf
conf = settings
setup_for_worker(celery, settings, prefix)
if db_connection:
# This creates the schema if it doesn't exist
initialize_db(TSBDBConnection, settings, allow_schema_create=True)
def test_parse_db_settings_with_no_generic_settings(self):
settings = {
'edware.db.dummyTenant.echo': 'True',
'edware.db.dummyTenant.schema_name': 'dummySchema',
'edware.db.dummyTenant.url': 'sqlite:///:memory:',
'edware.db.dummyTenant.state_code': 'NC',
'ignoreMe': 'setting',
'dummy': 'other settings'
}
initialize_db(EdCoreDBConnection, settings)
dbUtil = component.queryUtility(IDbUtil, 'edware.db.dummyTenant')
self.assertIsNotNone(dbUtil)
self.assertEqual(dbUtil.get_engine().echo, True)
self.assertEqual(dbUtil.get_metadata().schema,
settings['edware.db.dummyTenant.schema_name'])
self.assertEqual(dbUtil.get_engine().url.database, ':memory:')
def main(config_file, resource_dir, tenant_to_update, state_code, state_name, update_year):
'''
Imports data from csv
'''
config = configparser.ConfigParser()
config.read(config_file)
initialize_db(EdCoreDBConnection, config['app:main'])
for tenant in get_data_source_names():
if tenant_to_update in tenant:
delete_data(tenant)
import_csv_dir(resource_dir, tenant)
load_fact_asmt_outcome(tenant)
update_state(tenant, state_code, state_name)
if update_year:
update_aca_year(tenant)
def test_parse_db_settings_mapping(self):
settings = {'edware.db.echo': 'True',
'edware.db.schema_name': 'dummySchema',
'edware.db.dummyTenant.state_code': 'NC',
'edware.db.dummyTenant.url': 'sqlite:///:memory:',
'other': 'setting',
'dummy': 'other settings'}
mapping = initialize_db(EdCoreDBConnection, settings)
self.assertIsNotNone(mapping)
self.assertEqual('NC', mapping['dummyTenant'])
def test_parse_db_settings_mapping(self):
settings = {
'edware.db.echo': 'True',
'edware.db.schema_name': 'dummySchema',
'edware.db.dummyTenant.state_code': 'NC',
'edware.db.dummyTenant.url': 'sqlite:///:memory:',
'other': 'setting',
'dummy': 'other settings'
}
mapping = initialize_db(EdCoreDBConnection, settings)
self.assertIsNotNone(mapping)
self.assertEqual('NC', mapping['dummyTenant'])
def _setup_db_connection(settings):
'''
Initialize database connection
'''
initialize_db(EdMigratePublicConnection, settings)
initialize_db(EdMigratePrivateConnection, settings)
initialize_db(PublicDBConnection, settings)
def test_parse_db_settings_with_multi_tenancy(self):
settings = {
'edware.db.echo': 'False',
'edware.db.dummyTenant.echo': 'True',
'edware.db.schema_name': 'myname',
'edware.db.dummyTenant.schema_name': 'dummySchema',
'edware.db.dummyTenant.url': 'sqlite:///:memory:',
'edware.db.dummyTenant.state_code': 'NC',
'edware.db.aTenant.url': 'sqlite:///:memory:',
'edware.db.aTenant.state_code': 'AB',
'edware.db.bTenant.url': 'sqlite:///:memory:',
'edware.db.bTenant.state_code': 'AB',
'edware.db.bTenant.echo': 'True',
'ignoreMe': 'setting',
'dummy': 'other settings'
}
initialize_db(EdCoreDBConnection, settings)
self.assertEquals(len(get_data_source_names()), 3)
self.assertIn('edware.db.dummyTenant', get_data_source_names())
self.assertIn('edware.db.aTenant', get_data_source_names())
self.assertIn('edware.db.bTenant', get_data_source_names())
dbUtil = component.queryUtility(IDbUtil, 'edware.db.dummyTenant')
self.assertIsNotNone(dbUtil)
self.assertEqual(dbUtil.get_engine().echo, True)
self.assertEqual(dbUtil.get_metadata().schema,
settings['edware.db.dummyTenant.schema_name'])
self.assertEqual(dbUtil.get_engine().url.database, ':memory:')
dbUtil = component.queryUtility(IDbUtil, 'edware.db.aTenant')
self.assertIsNotNone(dbUtil)
self.assertEqual(dbUtil.get_engine().echo, False)
self.assertEqual(dbUtil.get_metadata().schema,
settings['edware.db.schema_name'])
self.assertEqual(dbUtil.get_engine().url.database, ':memory:')
dbUtil = component.queryUtility(IDbUtil, 'edware.db.bTenant')
self.assertIsNotNone(dbUtil)
self.assertEqual(dbUtil.get_engine().echo, True)
self.assertEqual(dbUtil.get_metadata().schema,
settings['edware.db.schema_name'])
self.assertEqual(dbUtil.get_engine().url.database, ':memory:')
def main(config_file, tenant_to_update, out_dir, verbose, raw, item, asmt_guid=None):
"""
Imports data from csv
"""
config = configparser.ConfigParser()
config.read(config_file)
initialize_db(EdCoreDBConnection, config['app:main'])
for tenant in get_data_source_names():
if tenant_to_update in tenant:
# Get necessary meta-data
state_code = get_state_code(tenant)
assessments = get_all_assessments(tenant, asmt_guid)
if item:
# Create item pools for each assessment
create_item_pools(assessments)
# Build files for each assessment
for asmt in assessments:
students = get_students_for_assessment(tenant, asmt['guid'])
generate_data_files(out_dir, state_code, asmt, students, verbose, raw, item)
def main():
'''
Main Entry for ad-hoc testing to trigger batcher
'''
parser = ArgumentParser(description='File Batcher entry point')
parser.add_argument('-p', dest='pid_file', default='/opt/edware/run/file-monitor-smarter_score_batcher.pid',
help="pid file for TSB file monitor daemon")
parser.add_argument('-d', dest='daemon', action='store_true', default=False,
help="daemon mode for TSB file monitor")
parser.add_argument('-i', dest='ini_file', default='/opt/edware/conf/smarter_score_batcher.ini', help="ini file")
args = parser.parse_args()
if args.daemon:
create_daemon(args.pid_file)
file = args.ini_file
logging.config.fileConfig(file)
settings = read_ini(file)
initialize_db(TSBDBConnection, settings)
if args.daemon:
run_cron_sync_file(settings)
while True:
time.sleep(1)
else:
move_to_staging(settings)
from edmigrate.database.repmgr_connector import RepMgrDBConnection
from edmigrate.utils.constants import Constants
logger = logging.getLogger(Constants.WORKER_NAME)
PREFIX = 'migrate.celery'
def setup_celery(settings, prefix=PREFIX):
'''
Setup celery based on parameters defined in setting (ini file).
This calls by client application when dictionary of settings is given
:param settings: dict of configurations
:param prefix: prefix in configurations used for configuring celery
'''
setup_for_worker(celery, settings, prefix)
setup_settings(settings)
# Create an instance of celery, check if it's for prod celeryd mode and configure it for prod mode if so
celery, conf = configure_celeryd(PREFIX, prefix=PREFIX)
prod_config = get_config_file()
if prod_config:
# We should only need to setup db connection in prod mode
# setup_db_connection(conf)
initialize_db(RepMgrDBConnection, conf)
initialize_db(StatsDBConnection, conf)
setup_settings(conf)
logging.config.fileConfig(prod_config)
def main(global_config, **settings):
"""
Starting point for the Smarter application. Prepares the environment and
applies configurations. Sets paths, permissions, routes. Calls includeme
methods of EdAuth, EdApi, and EdServices.
Returns a Pyramid WSGI application.
"""
mode = settings.get('mode', 'prod').upper()
# read remote config
if mode == 'PROD' and 'edware.remote_config.url' in settings:
url = settings['edware.remote_config.url']
settings = get_remote_config(url)
# Prepare for environment specific
set_environment_path_variable(settings)
prepare_env(settings)
# set beaker cache region
set_cache_regions_from_settings(settings)
config = Configurator(settings=settings, root_factory=RootFactory)
tenant_mapping = initialize_db(EdCoreDBConnection, settings)
tenant_mapping_public = initialize_db(PublicDBConnection, settings)
initialize_db(StatsDBConnection, settings, allow_schema_create=True)
# save tenancy mapping
set_tenant_map(tenant_mapping)
# save tenant mapping for public reports
set_tenant_map_public_reports(tenant_mapping_public)
year_back = settings['smarter.reports.year_back']
set_default_year_back(year_back)
# setup celery
setup_services_celery(settings, prefix=servicesPrefix)
setup_extract_celery(settings, prefix=edextractPrefix)
initialize_hpz(settings)
# include edauth. Calls includeme
config.include(edauth)
# Pass edauth the roles/permission mapping that is defined in smarter
edauth.set_roles(RootFactory.__acl__)
# include add routes from edapi. Calls includeme
config.include(edapi)
# Get absolute paths
here = os.path.abspath(os.path.dirname(__file__))
assets_dir = os.path.abspath(os.path.join(os.path.join(here, '..'), 'assets'))
static_max_age = int(settings.get('smarter.resources.static.max_age', 3600))
config.add_static_view('assets/css', os.path.join(assets_dir, 'css'), cache_max_age=static_max_age)
config.add_static_view('assets/data', os.path.join(assets_dir, 'data'), cache_max_age=static_max_age)
config.add_static_view('assets/images', os.path.join(assets_dir, 'images'), cache_max_age=static_max_age)
config.add_static_view('assets/js', os.path.join(assets_dir, 'js'), cache_max_age=static_max_age)
config.add_static_view('assets/html', os.path.join(assets_dir, 'html'), cache_max_age=static_max_age)
config.add_static_view('assets/public', os.path.join(assets_dir, 'public'), cache_max_age=static_max_age, permission=pyramid.security.NO_PERMISSION_REQUIRED)
# Only expose test in non-production modes
if mode != 'PROD':
config.add_static_view('assets/test', os.path.join(assets_dir, 'test'), cache_max_age=static_max_age)
disable_stack_trace = settings.get('disable_stack_trace', 'False').upper()
if disable_stack_trace == 'TRUE':
# we would like to disable the stack trace when we are in production mode
sys.tracebacklimit = 0
# route for error
config.add_route('error', '/assets/public/error.html')
# include add routes from smarter.services. Calls includeme
config.include(services)
# include add routes from smarter.trigger. Calls includeme
config.include(trigger)
# scans smarter
config.scan()
# Set default permission on all views
config.set_default_permission(Permission.VIEW)
logger.info("Smarter started")
return config.make_wsgi_app()
def test_initialize_db_with_no_config(self):
settings = {}
initialize_db(StatsDBConnection, settings)
self.assertEquals(len(get_data_source_names()), 0)
def initialize_dbs(run_migrate_only, settings):
initialize_db(StatsDBConnection, settings)
initialize_db(EdMigrateSourceConnection, settings)
initialize_db(EdMigrateDestConnection, settings)
if not run_migrate_only:
initialize_db(RepMgrDBConnection, settings)
from edcore.database import initialize_db
from edmigrate.database.repmgr_connector import RepMgrDBConnection
from edmigrate.utils.constants import Constants
logger = logging.getLogger(Constants.WORKER_NAME)
PREFIX = 'migrate.celery'
def setup_celery(settings, prefix=PREFIX):
'''
Setup celery based on parameters defined in setting (ini file).
This calls by client application when dictionary of settings is given
:param settings: dict of configurations
:param prefix: prefix in configurations used for configuring celery
'''
setup_for_worker(celery, settings, prefix)
setup_settings(settings)
# Create an instance of celery, check if it's for prod celeryd mode and configure it for prod mode if so
celery, conf = configure_celeryd(PREFIX, prefix=PREFIX)
prod_config = get_config_file()
if prod_config:
# We should only need to setup db connection in prod mode
# setup_db_connection(conf)
initialize_db(RepMgrDBConnection, conf)
initialize_db(StatsDBConnection, conf)
setup_settings(conf)
logging.config.fileConfig(prod_config)
def setup_db_connection(settings):
'''
Given ini key/value pair, initialize db for stats table
'''
initialize_db(EdCoreDBConnection, settings)
initialize_db(StatsDBConnection, settings, allow_schema_create=True)
def test_initialize_db_with_no_config(self):
settings = {}
initialize_db(StatsDBConnection, settings)
self.assertEquals(len(get_data_source_names()), 0)
def main(global_config, **settings):
"""
Starting point for the Smarter application. Prepares the environment and
applies configurations. Sets paths, permissions, routes. Calls includeme
methods of EdAuth, EdApi, and EdServices.
Returns a Pyramid WSGI application.
"""
mode = settings.get('mode', 'prod').upper()
# read remote config
if mode == 'PROD' and 'edware.remote_config.url' in settings:
url = settings['edware.remote_config.url']
settings = get_remote_config(url)
# Prepare for environment specific
set_environment_path_variable(settings)
prepare_env(settings)
# set beaker cache region
set_cache_regions_from_settings(settings)
config = Configurator(settings=settings, root_factory=RootFactory)
tenant_mapping = initialize_db(EdCoreDBConnection, settings)
tenant_mapping_public = initialize_db(PublicDBConnection, settings)
initialize_db(StatsDBConnection, settings, allow_schema_create=True)
# save tenancy mapping
set_tenant_map(tenant_mapping)
# save tenant mapping for public reports
set_tenant_map_public_reports(tenant_mapping_public)
year_back = settings['smarter.reports.year_back']
set_default_year_back(year_back)
# setup celery
setup_services_celery(settings, prefix=servicesPrefix)
setup_extract_celery(settings, prefix=edextractPrefix)
initialize_hpz(settings)
# include edauth. Calls includeme
config.include(edauth)
# Pass edauth the roles/permission mapping that is defined in smarter
edauth.set_roles(RootFactory.__acl__)
# include add routes from edapi. Calls includeme
config.include(edapi)
# Get absolute paths
here = os.path.abspath(os.path.dirname(__file__))
assets_dir = os.path.abspath(
os.path.join(os.path.join(here, '..'), 'assets'))
static_max_age = int(settings.get('smarter.resources.static.max_age',
3600))
config.add_static_view('assets/css',
os.path.join(assets_dir, 'css'),
cache_max_age=static_max_age)
config.add_static_view('assets/data',
os.path.join(assets_dir, 'data'),
cache_max_age=static_max_age)
config.add_static_view('assets/images',
os.path.join(assets_dir, 'images'),
cache_max_age=static_max_age)
config.add_static_view('assets/js',
os.path.join(assets_dir, 'js'),
cache_max_age=static_max_age)
config.add_static_view('assets/html',
os.path.join(assets_dir, 'html'),
cache_max_age=static_max_age)
config.add_static_view('assets/public',
os.path.join(assets_dir, 'public'),
cache_max_age=static_max_age,
permission=pyramid.security.NO_PERMISSION_REQUIRED)
# Only expose test in non-production modes
if mode != 'PROD':
config.add_static_view('assets/test',
os.path.join(assets_dir, 'test'),
cache_max_age=static_max_age)
disable_stack_trace = settings.get('disable_stack_trace', 'False').upper()
if disable_stack_trace == 'TRUE':
# we would like to disable the stack trace when we are in production mode
sys.tracebacklimit = 0
# route for error
config.add_route('error', '/assets/public/error.html')
# include add routes from smarter.services. Calls includeme
config.include(services)
# include add routes from smarter.trigger. Calls includeme
config.include(trigger)
# scans smarter
config.scan()
# Set default permission on all views
config.set_default_permission(Permission.VIEW)
logger.info("Smarter started")
return config.make_wsgi_app()
def initialize_dbs(run_migrate_only, settings):
initialize_db(StatsDBConnection, settings)
initialize_db(EdMigrateSourceConnection, settings)
initialize_db(EdMigrateDestConnection, settings)
if not run_migrate_only:
initialize_db(RepMgrDBConnection, settings)