def test_migrate_batch_stureg(self):
batch_guid = '2bb942b9-75cf-4055-a67a-8b9ab53a9dfc'
batch = {UdlStatsConstants.REC_ID: '6',
UdlStatsConstants.BATCH_GUID: batch_guid, UdlStatsConstants.TENANT: self.__tenant,
UdlStatsConstants.SCHEMA_NAME: None, Constants.DEACTIVATE: False,
UdlStatsConstants.LOAD_TYPE: LoadType.STUDENT_REGISTRATION,
UdlStatsConstants.BATCH_OPERATION: 's',
UdlStatsConstants.SNAPSHOT_CRITERIA: '{"reg_system_id": "015247bd-058c-48cd-bb4d-f6cffe5b40c1", "academic_year": 2015}'}
self.insert_into_udl_stats(batch[UdlStatsConstants.REC_ID], batch_guid, self.__tenant, batch[UdlStatsConstants.LOAD_TYPE])
preprod_conn = EdMigrateSourceConnection(tenant=get_unittest_preprod_tenant_name())
count_to_source_query = select([func.count()]).select_from(preprod_conn.get_table(Constants.STUDENT_REG))
count_to_be_inserted = preprod_conn.execute(count_to_source_query).fetchall()[0][0]
self.assertEqual(10, count_to_be_inserted)
prod_conn = EdMigrateDestConnection(tenant=get_unittest_preprod_tenant_name())
student_reg_table = prod_conn.get_table(Constants.STUDENT_REG)
count_query = select([func.count()]).select_from(student_reg_table)
count_before = prod_conn.execute(count_query).fetchall()[0][0]
self.assertEqual(2581, count_before)
count_snapshot_query = select([func.count()], student_reg_table.c.academic_year == 2015).select_from(student_reg_table)
count_to_be_deleted = prod_conn.execute(count_snapshot_query).fetchall()[0][0]
self.assertEqual(1217, count_to_be_deleted)
rtn = migrate_batch(batch)
self.assertTrue(rtn)
expected_count_after = count_before - count_to_be_deleted + count_to_be_inserted
count_after = prod_conn.execute(count_query).fetchall()[0][0]
self.assertEqual(expected_count_after, count_after)
def test_migrate_student_reg(self):
Unittest_with_edcore_sqlite.setUpClass(EdMigrateDestConnection.get_datasource_name(TestMigrate.test_tenant),
use_metadata_from_db=False)
preprod_conn = EdMigrateSourceConnection(tenant=get_unittest_preprod_tenant_name())
prod_conn = EdMigrateDestConnection(tenant=get_unittest_prod_tenant_name())
batch_guid = "0aa942b9-75cf-4055-a67a-8b9ab53a9dfc"
student_reg_table = preprod_conn.get_table(Constants.STUDENT_REG)
get_query = select([student_reg_table.c.student_reg_rec_id]).order_by(student_reg_table.c.student_reg_rec_id)
count_query = select([func.count().label('student_reg_rec_ids')],
student_reg_table.c.student_reg_rec_id.in_(range(15541, 15551)))
rset = preprod_conn.execute(get_query)
row = rset.fetchall()
self.assertEqual(10, len(row))
self.assertListEqual([(15541,), (15542,), (15543,), (15544,), (15545,), (15546,), (15547,), (15548,), (15549,), (15550,)],
row)
rset.close()
rset = prod_conn.execute(count_query)
row = rset.fetchone()
self.assertEqual(0, row['student_reg_rec_ids'])
rset.close()
delete_count, insert_count = migrate_table(batch_guid, None, preprod_conn, prod_conn, 'student_reg', False)
self.assertEqual(0, delete_count)
self.assertEqual(10, insert_count)
rset = prod_conn.execute(count_query)
row = rset.fetchone()
self.assertEqual(10, row['student_reg_rec_ids'])
rset.close()
def setUpClass(cls):
Unittest_with_edcore_sqlite.setUpClass(
EdMigrateDestConnection.get_datasource_name(
TestMigrate.test_tenant),
use_metadata_from_db=False)
Unittest_with_preprod_sqlite.setUpClass()
Unittest_with_stats_sqlite.setUpClass()
def get_prod_table_count(tenant, table_name):
'''
Check count numbers in prod tables
'''
with EdMigrateDestConnection(tenant) as conn:
table = conn.get_table(table_name)
query = select([count().label('total')], from_obj=[table])
results = conn.get_result(query)
return results[0]['total']
def test_migrate_fact_asmt_outcome_vw_record_already_deleted1(self):
preprod_conn = EdMigrateSourceConnection(
tenant=get_unittest_preprod_tenant_name())
prod_conn = EdMigrateDestConnection(
tenant=get_unittest_prod_tenant_name())
batch_guid = "288220EB-3876-41EB-B3A7-F0E6C8BD013B"
self.assertRaises(EdMigrateRecordAlreadyDeletedException,
migrate_table, batch_guid, None, preprod_conn,
prod_conn, 'fact_asmt_outcome_vw', False)
def test_migrate_student_reg(self):
Unittest_with_edcore_sqlite.setUpClass(
EdMigrateDestConnection.get_datasource_name(
TestMigrate.test_tenant),
use_metadata_from_db=False)
preprod_conn = EdMigrateSourceConnection(
tenant=get_unittest_preprod_tenant_name())
prod_conn = EdMigrateDestConnection(
tenant=get_unittest_prod_tenant_name())
batch_guid = "0aa942b9-75cf-4055-a67a-8b9ab53a9dfc"
student_reg_table = preprod_conn.get_table(Constants.STUDENT_REG)
get_query = select([student_reg_table.c.student_reg_rec_id
]).order_by(student_reg_table.c.student_reg_rec_id)
count_query = select([func.count().label('student_reg_rec_ids')],
student_reg_table.c.student_reg_rec_id.in_(
range(15541, 15551)))
rset = preprod_conn.execute(get_query)
row = rset.fetchall()
self.assertEqual(10, len(row))
self.assertListEqual([(15541, ), (15542, ), (15543, ), (15544, ),
(15545, ), (15546, ), (15547, ), (15548, ),
(15549, ), (15550, )], row)
rset.close()
rset = prod_conn.execute(count_query)
row = rset.fetchone()
self.assertEqual(0, row['student_reg_rec_ids'])
rset.close()
delete_count, insert_count = migrate_table(batch_guid, None,
preprod_conn, prod_conn,
'student_reg', False)
self.assertEqual(0, delete_count)
self.assertEqual(10, insert_count)
rset = prod_conn.execute(count_query)
row = rset.fetchone()
self.assertEqual(10, row['student_reg_rec_ids'])
rset.close()
def test_migrate_fact_asmt_outcome_vw(self):
preprod_conn = EdMigrateSourceConnection(tenant=get_unittest_preprod_tenant_name())
prod_conn = EdMigrateDestConnection(tenant=get_unittest_prod_tenant_name())
batch_guid = "288220EB-3876-41EB-B3A7-F0E6C8BD013B"
fact_asmt_outcome_table = prod_conn.get_table(Constants.FACT_ASMT_OUTCOME)
query = select([func.count().label('asmt_outcome_vw_rec_ids')], fact_asmt_outcome_table.c.asmt_outcome_vw_rec_id.in_([1000000776, 1000001034, 1000001112]))
query_c = query.where(fact_asmt_outcome_table.c.rec_status == 'C')
query_d = query.where(fact_asmt_outcome_table.c.rec_status == 'D')
query_I = query.where(fact_asmt_outcome_table.c.rec_status == 'I')
rset = prod_conn.execute(query_c)
row = rset.fetchone()
self.assertEqual(3, row['asmt_outcome_vw_rec_ids'])
rset.close()
delete_count, insert_count = migrate_table(batch_guid, None, preprod_conn, prod_conn,
'fact_asmt_outcome_vw', False)
self.assertEqual(3, delete_count)
self.assertEqual(3, insert_count)
rset = prod_conn.execute(query_c)
row = rset.fetchone()
self.assertEqual(0, row['asmt_outcome_vw_rec_ids'])
rset.close()
rset = prod_conn.execute(query_d)
row = rset.fetchone()
self.assertEqual(3, row['asmt_outcome_vw_rec_ids'])
rset.close()
# The deactivation count will be always zero in unit test
rset = prod_conn.execute(query_I)
row = rset.fetchone()
self.assertEqual(0, row['asmt_outcome_vw_rec_ids'])
rset.close()
def test_migrate_batch_with_roll_back(self):
batch = {UdlStatsConstants.REC_ID: '7',
UdlStatsConstants.BATCH_GUID: '13DCC2AB-4FC6-418D-844E-65ED5D9CED38',
UdlStatsConstants.TENANT: 'tomcat', UdlStatsConstants.SCHEMA_NAME: None,
Constants.DEACTIVATE: False, UdlStatsConstants.LOAD_TYPE: LoadType.ASSESSMENT,
UdlStatsConstants.BATCH_OPERATION: None,
UdlStatsConstants.SNAPSHOT_CRITERIA: None}
prod_conn = EdMigrateDestConnection(tenant=get_unittest_prod_tenant_name())
fact_asmt_outcome_table = prod_conn.get_table(Constants.FACT_ASMT_OUTCOME)
query = select([fact_asmt_outcome_table], fact_asmt_outcome_table.c.asmt_outcome_vw_rec_id.in_([1000000777, 1000000778, 1000001035]))
query_c = query.where(fact_asmt_outcome_table.c.rec_status == 'C')
query_d = query.where(fact_asmt_outcome_table.c.rec_status == 'D')
rset = prod_conn.execute(query_c)
rows = rset.fetchall()
row_cnt = len(rows)
self.assertEqual(3, row_cnt)
rset.close()
rset = prod_conn.execute(query_d)
rows = rset.fetchall()
row_cnt = len(rows)
self.assertEqual(0, row_cnt)
rtn = migrate_batch(batch)
rset.close()
self.assertFalse(rtn)
rset = prod_conn.execute(query_c)
rows = rset.fetchall()
row_cnt = len(rows)
self.assertEqual(3, row_cnt)
rset.close()
rset = prod_conn.execute(query_d)
rows = rset.fetchall()
row_cnt = len(rows)
self.assertEqual(0, row_cnt)
rtn = migrate_batch(batch)
rset.close()
def test_migrate_batch_stureg(self):
batch_guid = '2bb942b9-75cf-4055-a67a-8b9ab53a9dfc'
batch = {
UdlStatsConstants.REC_ID:
'6',
UdlStatsConstants.BATCH_GUID:
batch_guid,
UdlStatsConstants.TENANT:
self.__tenant,
UdlStatsConstants.SCHEMA_NAME:
None,
Constants.DEACTIVATE:
False,
UdlStatsConstants.LOAD_TYPE:
LoadType.STUDENT_REGISTRATION,
UdlStatsConstants.BATCH_OPERATION:
's',
UdlStatsConstants.SNAPSHOT_CRITERIA:
'{"reg_system_id": "015247bd-058c-48cd-bb4d-f6cffe5b40c1", "academic_year": 2015}'
}
self.insert_into_udl_stats(batch[UdlStatsConstants.REC_ID], batch_guid,
self.__tenant,
batch[UdlStatsConstants.LOAD_TYPE])
preprod_conn = EdMigrateSourceConnection(
tenant=get_unittest_preprod_tenant_name())
count_to_source_query = select([func.count()]).select_from(
preprod_conn.get_table(Constants.STUDENT_REG))
count_to_be_inserted = preprod_conn.execute(
count_to_source_query).fetchall()[0][0]
self.assertEqual(10, count_to_be_inserted)
prod_conn = EdMigrateDestConnection(
tenant=get_unittest_preprod_tenant_name())
student_reg_table = prod_conn.get_table(Constants.STUDENT_REG)
count_query = select([func.count()]).select_from(student_reg_table)
count_before = prod_conn.execute(count_query).fetchall()[0][0]
self.assertEqual(2581, count_before)
count_snapshot_query = select([func.count()],
student_reg_table.c.academic_year ==
2015).select_from(student_reg_table)
count_to_be_deleted = prod_conn.execute(
count_snapshot_query).fetchall()[0][0]
self.assertEqual(1217, count_to_be_deleted)
rtn = migrate_batch(batch)
self.assertTrue(rtn)
expected_count_after = count_before - count_to_be_deleted + count_to_be_inserted
count_after = prod_conn.execute(count_query).fetchall()[0][0]
self.assertEqual(expected_count_after, count_after)
def test_migrate_getting_natural_key(self):
with EdMigrateDestConnection(
tenant=get_unittest_prod_tenant_name()) as prod_conn:
self.assertEquals(
get_natural_key_columns(prod_conn.get_table('dim_student')),
['student_id'])
self.assertEquals(
get_natural_key_columns(prod_conn.get_table('dim_asmt')),
['asmt_guid'])
self.assertEquals(
get_natural_key_columns(
prod_conn.get_table('fact_asmt_outcome_vw')),
['asmt_guid', 'student_id', 'date_taken'])
self.assertEquals(
get_natural_key_columns(
prod_conn.get_table('fact_asmt_outcome')),
['asmt_guid', 'student_id', 'date_taken'])
self.assertEquals(
get_natural_key_columns(prod_conn.get_table('dim_inst_hier')),
['state_code', 'district_id', 'school_id'])
def test_migrate_batch_with_roll_back(self):
batch = {
UdlStatsConstants.REC_ID: '7',
UdlStatsConstants.BATCH_GUID:
'13DCC2AB-4FC6-418D-844E-65ED5D9CED38',
UdlStatsConstants.TENANT: 'tomcat',
UdlStatsConstants.SCHEMA_NAME: None,
Constants.DEACTIVATE: False,
UdlStatsConstants.LOAD_TYPE: LoadType.ASSESSMENT,
UdlStatsConstants.BATCH_OPERATION: None,
UdlStatsConstants.SNAPSHOT_CRITERIA: None
}
prod_conn = EdMigrateDestConnection(
tenant=get_unittest_prod_tenant_name())
fact_asmt_outcome_table = prod_conn.get_table(
Constants.FACT_ASMT_OUTCOME)
query = select([fact_asmt_outcome_table],
fact_asmt_outcome_table.c.asmt_outcome_vw_rec_id.in_(
[1000000777, 1000000778, 1000001035]))
query_c = query.where(fact_asmt_outcome_table.c.rec_status == 'C')
query_d = query.where(fact_asmt_outcome_table.c.rec_status == 'D')
rset = prod_conn.execute(query_c)
rows = rset.fetchall()
row_cnt = len(rows)
self.assertEqual(3, row_cnt)
rset.close()
rset = prod_conn.execute(query_d)
rows = rset.fetchall()
row_cnt = len(rows)
self.assertEqual(0, row_cnt)
rtn = migrate_batch(batch)
rset.close()
self.assertFalse(rtn)
rset = prod_conn.execute(query_c)
rows = rset.fetchall()
row_cnt = len(rows)
self.assertEqual(3, row_cnt)
rset.close()
rset = prod_conn.execute(query_d)
rows = rset.fetchall()
row_cnt = len(rows)
self.assertEqual(0, row_cnt)
rtn = migrate_batch(batch)
rset.close()
def test_migrate_fact_asmt_outcome_vw(self):
preprod_conn = EdMigrateSourceConnection(
tenant=get_unittest_preprod_tenant_name())
prod_conn = EdMigrateDestConnection(
tenant=get_unittest_prod_tenant_name())
batch_guid = "288220EB-3876-41EB-B3A7-F0E6C8BD013B"
fact_asmt_outcome_table = prod_conn.get_table(
Constants.FACT_ASMT_OUTCOME)
query = select([func.count().label('asmt_outcome_vw_rec_ids')],
fact_asmt_outcome_table.c.asmt_outcome_vw_rec_id.in_(
[1000000776, 1000001034, 1000001112]))
query_c = query.where(fact_asmt_outcome_table.c.rec_status == 'C')
query_d = query.where(fact_asmt_outcome_table.c.rec_status == 'D')
query_I = query.where(fact_asmt_outcome_table.c.rec_status == 'I')
rset = prod_conn.execute(query_c)
row = rset.fetchone()
self.assertEqual(3, row['asmt_outcome_vw_rec_ids'])
rset.close()
delete_count, insert_count = migrate_table(batch_guid, None,
preprod_conn, prod_conn,
'fact_asmt_outcome_vw',
False)
self.assertEqual(3, delete_count)
self.assertEqual(3, insert_count)
rset = prod_conn.execute(query_c)
row = rset.fetchone()
self.assertEqual(0, row['asmt_outcome_vw_rec_ids'])
rset.close()
rset = prod_conn.execute(query_d)
row = rset.fetchone()
self.assertEqual(3, row['asmt_outcome_vw_rec_ids'])
rset.close()
# The deactivation count will be always zero in unit test
rset = prod_conn.execute(query_I)
row = rset.fetchone()
self.assertEqual(0, row['asmt_outcome_vw_rec_ids'])
rset.close()
def migrate_batch(batch):
"""Migrates data for the given batch and given tenant
:param batch_guid: Batch Guid of the batch under migration
:returns true: sucess, false: fail (for UT purpose)
"""
rtn = False
rec_id = batch[UdlStatsConstants.REC_ID]
batch_guid = batch[UdlStatsConstants.BATCH_GUID]
tenant = batch[UdlStatsConstants.TENANT]
schema_name = batch[UdlStatsConstants.SCHEMA_NAME]
load_type = batch[UdlStatsConstants.LOAD_TYPE]
batch_op = batch[UdlStatsConstants.BATCH_OPERATION]
batch_criteria = batch[UdlStatsConstants.SNAPSHOT_CRITERIA]
# this flag will be set to false from unit test, if this is not set its always True
deactivate = batch[
Constants.DEACTIVATE] if Constants.DEACTIVATE in batch else True
logger.info('Migrating batch: ' + batch_guid + ',for tenant: ' + tenant)
with EdMigrateDestConnection(tenant) as dest_connector, \
EdMigrateSourceConnection(tenant) as source_connector:
try:
# start transaction for this batch
trans = dest_connector.get_transaction()
source_connector.set_metadata_by_generate(
schema_name=schema_name, metadata_func=generate_ed_metadata)
report_udl_stats_batch_status(rec_id,
UdlStatsConstants.MIGRATE_IN_PROCESS)
tables_to_migrate = get_ordered_tables_to_migrate(
dest_connector, load_type)
# migrate all tables
migrate_all_tables(batch_guid,
schema_name,
source_connector,
dest_connector,
tables_to_migrate,
deactivate=deactivate,
batch_op=batch_op,
batch_criteria=batch_criteria)
# report udl stats with the new batch migrated
report_udl_stats_batch_status(rec_id,
UdlStatsConstants.MIGRATE_INGESTED)
# commit transaction
trans.commit()
logger.info('Master: Migration successful for batch: ' +
batch_guid)
rtn = True
except Exception as e:
logger.info('Exception happened while migrating batch: ' +
batch_guid + ' - Rollback initiated')
logger.info(e)
print(e)
logger.exception('migrate rollback')
trans.rollback()
try:
report_udl_stats_batch_status(rec_id,
UdlStatsConstants.MIGRATE_FAILED)
except Exception as e:
pass
return rtn
def setUpClass(cls):
Unittest_with_edcore_sqlite.setUpClass(EdMigrateDestConnection.get_datasource_name(TestMigrate.test_tenant),
use_metadata_from_db=False)
Unittest_with_preprod_sqlite.setUpClass()
Unittest_with_stats_sqlite.setUpClass()