def test_migrate_batch_stureg(self):
batch_guid = '2bb942b9-75cf-4055-a67a-8b9ab53a9dfc'
batch = {UdlStatsConstants.REC_ID: '6',
UdlStatsConstants.BATCH_GUID: batch_guid, UdlStatsConstants.TENANT: self.__tenant,
UdlStatsConstants.SCHEMA_NAME: None, Constants.DEACTIVATE: False,
UdlStatsConstants.LOAD_TYPE: LoadType.STUDENT_REGISTRATION,
UdlStatsConstants.BATCH_OPERATION: 's',
UdlStatsConstants.SNAPSHOT_CRITERIA: '{"reg_system_id": "015247bd-058c-48cd-bb4d-f6cffe5b40c1", "academic_year": 2015}'}
self.insert_into_udl_stats(batch[UdlStatsConstants.REC_ID], batch_guid, self.__tenant, batch[UdlStatsConstants.LOAD_TYPE])
preprod_conn = EdMigrateSourceConnection(tenant=get_unittest_preprod_tenant_name())
count_to_source_query = select([func.count()]).select_from(preprod_conn.get_table(Constants.STUDENT_REG))
count_to_be_inserted = preprod_conn.execute(count_to_source_query).fetchall()[0][0]
self.assertEqual(10, count_to_be_inserted)
prod_conn = EdMigrateDestConnection(tenant=get_unittest_preprod_tenant_name())
student_reg_table = prod_conn.get_table(Constants.STUDENT_REG)
count_query = select([func.count()]).select_from(student_reg_table)
count_before = prod_conn.execute(count_query).fetchall()[0][0]
self.assertEqual(2581, count_before)
count_snapshot_query = select([func.count()], student_reg_table.c.academic_year == 2015).select_from(student_reg_table)
count_to_be_deleted = prod_conn.execute(count_snapshot_query).fetchall()[0][0]
self.assertEqual(1217, count_to_be_deleted)
rtn = migrate_batch(batch)
self.assertTrue(rtn)
expected_count_after = count_before - count_to_be_deleted + count_to_be_inserted
count_after = prod_conn.execute(count_query).fetchall()[0][0]
self.assertEqual(expected_count_after, count_after)
def test_migrate_student_reg(self):
Unittest_with_edcore_sqlite.setUpClass(EdMigrateDestConnection.get_datasource_name(TestMigrate.test_tenant),
use_metadata_from_db=False)
preprod_conn = EdMigrateSourceConnection(tenant=get_unittest_preprod_tenant_name())
prod_conn = EdMigrateDestConnection(tenant=get_unittest_prod_tenant_name())
batch_guid = "0aa942b9-75cf-4055-a67a-8b9ab53a9dfc"
student_reg_table = preprod_conn.get_table(Constants.STUDENT_REG)
get_query = select([student_reg_table.c.student_reg_rec_id]).order_by(student_reg_table.c.student_reg_rec_id)
count_query = select([func.count().label('student_reg_rec_ids')],
student_reg_table.c.student_reg_rec_id.in_(range(15541, 15551)))
rset = preprod_conn.execute(get_query)
row = rset.fetchall()
self.assertEqual(10, len(row))
self.assertListEqual([(15541,), (15542,), (15543,), (15544,), (15545,), (15546,), (15547,), (15548,), (15549,), (15550,)],
row)
rset.close()
rset = prod_conn.execute(count_query)
row = rset.fetchone()
self.assertEqual(0, row['student_reg_rec_ids'])
rset.close()
delete_count, insert_count = migrate_table(batch_guid, None, preprod_conn, prod_conn, 'student_reg', False)
self.assertEqual(0, delete_count)
self.assertEqual(10, insert_count)
rset = prod_conn.execute(count_query)
row = rset.fetchone()
self.assertEqual(10, row['student_reg_rec_ids'])
rset.close()
def test_migrate_batch_stureg(self):
batch_guid = '2bb942b9-75cf-4055-a67a-8b9ab53a9dfc'
batch = {
UdlStatsConstants.REC_ID:
'6',
UdlStatsConstants.BATCH_GUID:
batch_guid,
UdlStatsConstants.TENANT:
self.__tenant,
UdlStatsConstants.SCHEMA_NAME:
None,
Constants.DEACTIVATE:
False,
UdlStatsConstants.LOAD_TYPE:
LoadType.STUDENT_REGISTRATION,
UdlStatsConstants.BATCH_OPERATION:
's',
UdlStatsConstants.SNAPSHOT_CRITERIA:
'{"reg_system_id": "015247bd-058c-48cd-bb4d-f6cffe5b40c1", "academic_year": 2015}'
}
self.insert_into_udl_stats(batch[UdlStatsConstants.REC_ID], batch_guid,
self.__tenant,
batch[UdlStatsConstants.LOAD_TYPE])
preprod_conn = EdMigrateSourceConnection(
tenant=get_unittest_preprod_tenant_name())
count_to_source_query = select([func.count()]).select_from(
preprod_conn.get_table(Constants.STUDENT_REG))
count_to_be_inserted = preprod_conn.execute(
count_to_source_query).fetchall()[0][0]
self.assertEqual(10, count_to_be_inserted)
prod_conn = EdMigrateDestConnection(
tenant=get_unittest_preprod_tenant_name())
student_reg_table = prod_conn.get_table(Constants.STUDENT_REG)
count_query = select([func.count()]).select_from(student_reg_table)
count_before = prod_conn.execute(count_query).fetchall()[0][0]
self.assertEqual(2581, count_before)
count_snapshot_query = select([func.count()],
student_reg_table.c.academic_year ==
2015).select_from(student_reg_table)
count_to_be_deleted = prod_conn.execute(
count_snapshot_query).fetchall()[0][0]
self.assertEqual(1217, count_to_be_deleted)
rtn = migrate_batch(batch)
self.assertTrue(rtn)
expected_count_after = count_before - count_to_be_deleted + count_to_be_inserted
count_after = prod_conn.execute(count_query).fetchall()[0][0]
self.assertEqual(expected_count_after, count_after)
def setUpClass(cls):
here = os.path.abspath(os.path.dirname(__file__))
resources_dir = os.path.abspath(
os.path.join(os.path.join(here, '..', 'resources')))
super().setUpClass(EdMigrateSourceConnection.get_datasource_name(
get_unittest_tenant_name()),
resources_dir=resources_dir)
def test_migrate_fact_asmt_outcome_vw_record_already_deleted1(self):
preprod_conn = EdMigrateSourceConnection(
tenant=get_unittest_preprod_tenant_name())
prod_conn = EdMigrateDestConnection(
tenant=get_unittest_prod_tenant_name())
batch_guid = "288220EB-3876-41EB-B3A7-F0E6C8BD013B"
self.assertRaises(EdMigrateRecordAlreadyDeletedException,
migrate_table, batch_guid, None, preprod_conn,
prod_conn, 'fact_asmt_outcome_vw', False)
def test_migrate_student_reg(self):
Unittest_with_edcore_sqlite.setUpClass(
EdMigrateDestConnection.get_datasource_name(
TestMigrate.test_tenant),
use_metadata_from_db=False)
preprod_conn = EdMigrateSourceConnection(
tenant=get_unittest_preprod_tenant_name())
prod_conn = EdMigrateDestConnection(
tenant=get_unittest_prod_tenant_name())
batch_guid = "0aa942b9-75cf-4055-a67a-8b9ab53a9dfc"
student_reg_table = preprod_conn.get_table(Constants.STUDENT_REG)
get_query = select([student_reg_table.c.student_reg_rec_id
]).order_by(student_reg_table.c.student_reg_rec_id)
count_query = select([func.count().label('student_reg_rec_ids')],
student_reg_table.c.student_reg_rec_id.in_(
range(15541, 15551)))
rset = preprod_conn.execute(get_query)
row = rset.fetchall()
self.assertEqual(10, len(row))
self.assertListEqual([(15541, ), (15542, ), (15543, ), (15544, ),
(15545, ), (15546, ), (15547, ), (15548, ),
(15549, ), (15550, )], row)
rset.close()
rset = prod_conn.execute(count_query)
row = rset.fetchone()
self.assertEqual(0, row['student_reg_rec_ids'])
rset.close()
delete_count, insert_count = migrate_table(batch_guid, None,
preprod_conn, prod_conn,
'student_reg', False)
self.assertEqual(0, delete_count)
self.assertEqual(10, insert_count)
rset = prod_conn.execute(count_query)
row = rset.fetchone()
self.assertEqual(10, row['student_reg_rec_ids'])
rset.close()
def cleanup_batch(batch):
"""Cleanup the pre-prod schema for the given batch
:param batch_guid: Batch Guid of the batch under migration
:returns true: sucess, false: fail (for UT purpose)
"""
rtn = False
batch_guid = batch[UdlStatsConstants.BATCH_GUID]
tenant = batch[UdlStatsConstants.TENANT]
schema_name = batch[UdlStatsConstants.SCHEMA_NAME]
logger.info('Cleaning up batch: ' + batch_guid + ',for tenant: ' + tenant)
with EdMigrateSourceConnection(tenant) as source_connector:
try:
drop_schema(source_connector, schema_name)
logger.info('Master: Cleanup successful for batch: ' + batch_guid)
rtn = True
except Exception as e:
logger.info('Exception happened while cleaning up batch: ' +
batch_guid)
logger.info(e)
return rtn
def test_migrate_fact_asmt_outcome_vw(self):
preprod_conn = EdMigrateSourceConnection(
tenant=get_unittest_preprod_tenant_name())
prod_conn = EdMigrateDestConnection(
tenant=get_unittest_prod_tenant_name())
batch_guid = "288220EB-3876-41EB-B3A7-F0E6C8BD013B"
fact_asmt_outcome_table = prod_conn.get_table(
Constants.FACT_ASMT_OUTCOME)
query = select([func.count().label('asmt_outcome_vw_rec_ids')],
fact_asmt_outcome_table.c.asmt_outcome_vw_rec_id.in_(
[1000000776, 1000001034, 1000001112]))
query_c = query.where(fact_asmt_outcome_table.c.rec_status == 'C')
query_d = query.where(fact_asmt_outcome_table.c.rec_status == 'D')
query_I = query.where(fact_asmt_outcome_table.c.rec_status == 'I')
rset = prod_conn.execute(query_c)
row = rset.fetchone()
self.assertEqual(3, row['asmt_outcome_vw_rec_ids'])
rset.close()
delete_count, insert_count = migrate_table(batch_guid, None,
preprod_conn, prod_conn,
'fact_asmt_outcome_vw',
False)
self.assertEqual(3, delete_count)
self.assertEqual(3, insert_count)
rset = prod_conn.execute(query_c)
row = rset.fetchone()
self.assertEqual(0, row['asmt_outcome_vw_rec_ids'])
rset.close()
rset = prod_conn.execute(query_d)
row = rset.fetchone()
self.assertEqual(3, row['asmt_outcome_vw_rec_ids'])
rset.close()
# The deactivation count will be always zero in unit test
rset = prod_conn.execute(query_I)
row = rset.fetchone()
self.assertEqual(0, row['asmt_outcome_vw_rec_ids'])
rset.close()
def setUpClass(cls):
super().setUpClass(
EdMigrateSourceConnection.get_datasource_name(
get_unittest_tenant_name()))
def migrate_batch(batch):
"""Migrates data for the given batch and given tenant
:param batch_guid: Batch Guid of the batch under migration
:returns true: sucess, false: fail (for UT purpose)
"""
rtn = False
rec_id = batch[UdlStatsConstants.REC_ID]
batch_guid = batch[UdlStatsConstants.BATCH_GUID]
tenant = batch[UdlStatsConstants.TENANT]
schema_name = batch[UdlStatsConstants.SCHEMA_NAME]
load_type = batch[UdlStatsConstants.LOAD_TYPE]
batch_op = batch[UdlStatsConstants.BATCH_OPERATION]
batch_criteria = batch[UdlStatsConstants.SNAPSHOT_CRITERIA]
# this flag will be set to false from unit test, if this is not set its always True
deactivate = batch[
Constants.DEACTIVATE] if Constants.DEACTIVATE in batch else True
logger.info('Migrating batch: ' + batch_guid + ',for tenant: ' + tenant)
with EdMigrateDestConnection(tenant) as dest_connector, \
EdMigrateSourceConnection(tenant) as source_connector:
try:
# start transaction for this batch
trans = dest_connector.get_transaction()
source_connector.set_metadata_by_generate(
schema_name=schema_name, metadata_func=generate_ed_metadata)
report_udl_stats_batch_status(rec_id,
UdlStatsConstants.MIGRATE_IN_PROCESS)
tables_to_migrate = get_ordered_tables_to_migrate(
dest_connector, load_type)
# migrate all tables
migrate_all_tables(batch_guid,
schema_name,
source_connector,
dest_connector,
tables_to_migrate,
deactivate=deactivate,
batch_op=batch_op,
batch_criteria=batch_criteria)
# report udl stats with the new batch migrated
report_udl_stats_batch_status(rec_id,
UdlStatsConstants.MIGRATE_INGESTED)
# commit transaction
trans.commit()
logger.info('Master: Migration successful for batch: ' +
batch_guid)
rtn = True
except Exception as e:
logger.info('Exception happened while migrating batch: ' +
batch_guid + ' - Rollback initiated')
logger.info(e)
print(e)
logger.exception('migrate rollback')
trans.rollback()
try:
report_udl_stats_batch_status(rec_id,
UdlStatsConstants.MIGRATE_FAILED)
except Exception as e:
pass
return rtn
def setUpClass(cls):
super().setUpClass(EdMigrateSourceConnection.get_datasource_name(get_unittest_tenant_name()))
def setUpClass(cls):
here = os.path.abspath(os.path.dirname(__file__))
resources_dir = os.path.abspath(os.path.join(os.path.join(here, '..', 'resources')))
super().setUpClass(EdMigrateSourceConnection.get_datasource_name(get_unittest_tenant_name()), resources_dir=resources_dir)