def test_create_schema_with_projections_and_overwrite(self):
task = self.create_task(cls=CopyToVerticaDummyTableWithProjections,
overwrite=True)
task.run()
mock_cursor = self.mock_vertica_connector.connect.return_value.cursor.return_value
expected = [
call("CREATE SCHEMA IF NOT EXISTS foobar"),
call(
"CREATE TABLE IF NOT EXISTS foobar.dummy_table "
"(id AUTO_INCREMENT,course_id VARCHAR(255),"
"interval_start DATETIME,interval_end DATETIME,label VARCHAR(255),"
"count INT,created TIMESTAMP DEFAULT NOW(),PRIMARY KEY (id))"),
call(
'CREATE PROJECTION IF NOT EXISTS foobar.dummy_table_projection_1 DEFINITION_1 on foobar.dummy_table;'
),
call(
'CREATE PROJECTION IF NOT EXISTS foobar.dummy_table_projection_3 DEFINITION_3 on foobar.dummy_table;'
),
call('DROP PROJECTION IF EXISTS foobar.dummy_table_projection_2;'),
call('DELETE FROM foobar.dummy_table'),
call(
"DELETE FROM foobar.name_of_marker_table where target_table='foobar.dummy_table';"
),
call("SELECT PURGE_TABLE('foobar.dummy_table')"),
call(
'CREATE PROJECTION IF NOT EXISTS foobar.dummy_table_projection_2 DEFINITION_2 on foobar.dummy_table;'
),
call('SELECT start_refresh();'),
]
self.assertEquals(expected, mock_cursor.execute.mock_calls)
def test_run_with_failure(self):
task = self.create_task()
task.output().touch = MagicMock(side_effect=Exception("Failed to update marker"))
with self.assertRaises(Exception):
task.run()
self.assertTrue(self.mock_mysql_connector.connect().cursor().execute.called)
self.assertTrue(self.mock_mysql_connector.connect().rollback.called)
self.assertFalse(self.mock_mysql_connector.connect().commit.called)
self.assertTrue(self.mock_mysql_connector.connect().close.called)
def test_run_with_failure(self):
task = self.create_task()
task.output().touch = MagicMock(side_effect=Exception("Failed to update marker"))
with self.assertRaises(Exception):
task.run()
self.assertTrue(self.mock_mysql_connector.connect().cursor().execute.called)
self.assertTrue(self.mock_mysql_connector.connect().rollback.called)
self.assertFalse(self.mock_mysql_connector.connect().commit.called)
self.assertTrue(self.mock_mysql_connector.connect().close.called)
def test_create_schema(self):
task = self.create_task()
task.run()
mock_cursor = self.mock_vertica_connector.connect.return_value.cursor.return_value
expected = [
call("CREATE SCHEMA IF NOT EXISTS foobar"),
call(
"CREATE TABLE IF NOT EXISTS foobar.dummy_table "
"(id AUTO_INCREMENT,course_id VARCHAR(255),"
"interval_start DATETIME,interval_end DATETIME,label VARCHAR(255),"
"count INT,created TIMESTAMP DEFAULT NOW(),PRIMARY KEY (id))"),
]
self.assertEquals(expected, mock_cursor.execute.mock_calls)
def test_create_database(self):
task = self.create_task()
task.run()
mock_cursor = self.mock_mysql_connector.connect.return_value.cursor.return_value
mock_cursor.execute.assert_has_calls([
call("CREATE DATABASE IF NOT EXISTS foobar"),
call(
"CREATE TABLE IF NOT EXISTS dummy_table "
"(id BIGINT(20) NOT NULL AUTO_INCREMENT,course_id VARCHAR(255),"
"interval_start DATETIME,interval_end DATETIME,label VARCHAR(255),"
"count INT,created TIMESTAMP DEFAULT NOW(),PRIMARY KEY (id))")
])
def test_create_database(self):
task = self.create_task()
task.run()
mock_cursor = self.mock_mysql_connector.connect.return_value.cursor.return_value
mock_cursor.execute.assert_has_calls([
call("CREATE DATABASE IF NOT EXISTS foobar"),
call(
"CREATE TABLE IF NOT EXISTS dummy_table "
"(id BIGINT(20) NOT NULL AUTO_INCREMENT,course_id VARCHAR(255),"
"interval_start DATETIME,interval_end DATETIME,label VARCHAR(255),"
"count INT,created TIMESTAMP DEFAULT NOW(),PRIMARY KEY (id))"
)
])
def test_create_schema(self):
task = self.create_task()
task.run()
mock_cursor = self.mock_vertica_connector.connect.return_value.cursor.return_value
mock_cursor.execute.assert_has_calls([
call("CREATE SCHEMA IF NOT EXISTS foobar"),
call(
"CREATE TABLE IF NOT EXISTS foobar.dummy_table "
"(id AUTO_INCREMENT,course_id VARCHAR(255),"
"interval_start DATETIME,interval_end DATETIME,label VARCHAR(255),"
"count INT,created TIMESTAMP DEFAULT NOW(),PRIMARY KEY (id))"
)
])
def test_create_schema_with_partitions(self):
task = self.create_task(cls=CopyToVerticaDummyTableWithPartitions)
task.run()
mock_cursor = self.mock_vertica_connector.connect.return_value.cursor.return_value
expected = [
call("CREATE SCHEMA IF NOT EXISTS foobar"),
call(
"CREATE TABLE IF NOT EXISTS foobar.dummy_table "
"(id AUTO_INCREMENT,course_id VARCHAR(255),"
"interval_start DATETIME,interval_end DATETIME,label VARCHAR(255),"
"count INT,created TIMESTAMP DEFAULT NOW(),PRIMARY KEY (id))"
" PARTITION BY course_id"),
call("SET TIMEZONE TO 'GMT';"),
]
self.assertEquals(expected, mock_cursor.execute.mock_calls)
def test_create_schema_with_partitions(self):
task = self.create_task(cls=CopyToVerticaDummyTableWithPartitions)
task.run()
mock_cursor = self.mock_vertica_connector.connect.return_value.cursor.return_value
expected = [
call("CREATE SCHEMA IF NOT EXISTS foobar"),
call(
"CREATE TABLE IF NOT EXISTS foobar.dummy_table "
"(id AUTO_INCREMENT,course_id VARCHAR(255),"
"interval_start DATETIME,interval_end DATETIME,label VARCHAR(255),"
"count INT,created TIMESTAMP DEFAULT NOW(),PRIMARY KEY (id))"
" PARTITION BY course_id"
),
call("SET TIMEZONE TO 'GMT';"),
]
self.assertEquals(expected, mock_cursor.execute.mock_calls)
def test_run(self):
task = self.create_task()
list(task.run())
self.assertTrue(self.mock_mysql_connector.connect().cursor().execute.called)
self.assertFalse(self.mock_mysql_connector.connect().rollback.called)
self.assertTrue(self.mock_mysql_connector.connect().commit.called)
self.assertTrue(self.mock_mysql_connector.connect().close.called)
def test_run(self):
task = self.create_task()
list(task.run())
self.assertTrue(
self.mock_mysql_connector.connect().cursor().execute.called)
self.assertFalse(self.mock_mysql_connector.connect().rollback.called)
self.assertTrue(self.mock_mysql_connector.connect().commit.called)
self.assertTrue(self.mock_mysql_connector.connect().close.called)
def test_create_schema_with_projections_and_no_overwrite(self):
task = self.create_task(cls=CopyToVerticaDummyTableWithProjections)
task.run()
mock_cursor = self.mock_vertica_connector.connect.return_value.cursor.return_value
expected = [
call("CREATE SCHEMA IF NOT EXISTS foobar"),
call(
"CREATE TABLE IF NOT EXISTS foobar.dummy_table "
"(id AUTO_INCREMENT,course_id VARCHAR(255),"
"interval_start DATETIME,interval_end DATETIME,label VARCHAR(255),"
"count INT,created TIMESTAMP DEFAULT NOW(),PRIMARY KEY (id))"
),
call('CREATE PROJECTION IF NOT EXISTS foobar.dummy_table_projection_1 DEFINITION_1 on foobar.dummy_table;'),
call('CREATE PROJECTION IF NOT EXISTS foobar.dummy_table_projection_3 DEFINITION_3 on foobar.dummy_table;'),
call("SET TIMEZONE TO 'GMT';"),
call('CREATE PROJECTION IF NOT EXISTS foobar.dummy_table_projection_2 DEFINITION_2 on foobar.dummy_table;'),
call('SELECT start_refresh();'),
]
self.assertEquals(expected, mock_cursor.execute.mock_calls)