def setUp(self):
self.adapter = Adapter()
self.mysql = {
"driver": "mysql",
"username": "******",
"password": "******",
"host": "localhost",
"port": 3066,
"database": "testdbfortests"
}
self.postgresql = {
"driver": "postgresql",
"username": "******",
"password": "******",
"host": "localhost",
"port": 5432,
"database": "testdb"
}
self.wrong = {
"driver": "wrongdriver",
"username": "******",
"password": "******",
"host": "localhost",
"port": 3066,
"database": "testdb"
}
class Database(object):
def __init__(self, connection_details):
self.connection_details = connection_details
self.adapter = Adapter()
self.adapter.setup(self.connection_details)
def get_tables(self):
return self.adapter.get_tables()
def get_columns(self, table):
return self.adapter.get_columns(table)
def get_top_rows(self, table):
return self.adapter.get_top_rows(table)
def setUp(self):
self.adapter = Adapter()
self.mysql = {
"driver": "mysql",
"username": "******",
"password": "******",
"host": "localhost",
"port": 3066,
"database": "testdbfortests"
}
self.postgresql = {
"driver": "postgresql",
"username": "******",
"password": "******",
"host": "localhost",
"port": 5432,
"database": "testdb"
}
self.wrong = {
"driver": "wrongdriver",
"username": "******",
"password": "******",
"host": "localhost",
"port": 3066,
"database": "testdb"
}
class Database(object): def __init__(self, connection_details): self.connection_details = connection_details self.adapter = Adapter() self.adapter.setup(self.connection_details) def get_tables(self): return self.adapter.get_tables() def get_columns(self, table): return self.adapter.get_columns(table) def get_top_rows(self, table): return self.adapter.get_top_rows(table)
def setUp(self):
self.id = "5073d616dd0ef409aaaaeeec0fee"
self.mysql = {
"driver": "mysql",
"username": "******",
"password": "******",
"host": "localhost",
"port": 3066,
"database": "testdbfortests"
}
self.adapter = Adapter()
self.adapter.setup(self.mysql)
self.details = mongo_backend.get_collection("details")
self.detail = self.details.find_one({"_id": ObjectId(self.id)})
self.sync = Sync(self.id)
class TestAdapter(object):
def setUp(self):
self.adapter = Adapter()
self.mysql = {
"driver": "mysql",
"username": "******",
"password": "******",
"host": "localhost",
"port": 3066,
"database": "testdbfortests"
}
self.postgresql = {
"driver": "postgresql",
"username": "******",
"password": "******",
"host": "localhost",
"port": 5432,
"database": "testdb"
}
self.wrong = {
"driver": "wrongdriver",
"username": "******",
"password": "******",
"host": "localhost",
"port": 3066,
"database": "testdb"
}
def test_create_connection_string(self):
# mysql connection string
eq_(self.adapter.create_connection_string(self.mysql),
"mysql://*****:*****@localhost:3066/testdbfortests")
# postgresql connection string
eq_(self.adapter.create_connection_string(self.postgresql),
"postgresql://*****:*****@localhost:5432/testdb")
def test_get_engine_valid_parameters(self):
# mysql connection string
self.adapter.connection_string = self.adapter.create_connection_string(
self.mysql)
assert self.adapter.get_engine() is not None
# postgresql connection string
self.adapter.connection_string = self.adapter.create_connection_string(
self.postgresql)
assert self.adapter.get_engine() is not None
# wrong connection string
self.adapter.connection_string = self.adapter.create_connection_string(
self.wrong)
assert self.adapter.get_engine() is None
def test_get_engine_metadata(self):
# mysql connection string
self.adapter.connection_string = self.adapter.create_connection_string(
self.mysql)
self.adapter.engine = self.adapter.get_engine()
assert self.adapter.get_engine_metadata(self.adapter.engine,
"reflect=True") is not None
# wrong connection string
self.adapter.connection_string = self.adapter.create_connection_string(
self.wrong)
self.adapter.engine = self.adapter.get_engine()
assert self.adapter.get_engine_metadata(self.adapter.engine,
"reflect=True") is None
def test_is_valid_connection(self):
# mysql connection string
self.connect(self.mysql)
assert self.adapter.engine and self.adapter.metadata
# postgresql connection string
self.connect(self.postgresql)
assert self.adapter.engine and self.adapter.metadata
# wrong connection string
self.connect(self.wrong)
assert not (self.adapter.engine and self.adapter.metadata)
def connect(self, connection_string):
self.adapter.connection_string = self.adapter.create_connection_string(
connection_string)
self.adapter.engine = self.adapter.get_engine()
self.adapter.metadata = self.adapter.get_engine_metadata(
self.adapter.engine, "reflect=True")
def test_get_tables(self):
self.connect(self.mysql)
tables = self.adapter.get_tables()
assert (tables is not None)
assert (len(tables) == 1)
assert (tables[0] == "user")
def test_get_columns(self):
self.connect(self.mysql)
columns = self.adapter.get_columns("user")
assert (columns is not None)
assert (len(columns) == 4)
assert (columns[0] == "userid")
assert (columns[1] == "username")
assert (columns[2] == "email_address")
assert (columns[3] == "date_created")
def test_get_rows(self):
# mysql connection string
self.connect(self.mysql)
rows = self.adapter.get_top_rows("user")
assert (rows is not None)
assert (len(rows) == 3)
assert (len(rows[0]) == 4)
assert (len(rows[1]) == 4)
assert (len(rows[2]) == 4)
assert (rows[0][0] == 1)
assert (rows[0][1] == "John")
assert (rows[0][2] == "*****@*****.**")
assert (rows[0][3] is None)
assert (rows[1][0] == 2)
assert (rows[1][1] == "Susan")
assert (rows[1][2] == "*****@*****.**")
assert (rows[1][3] is None)
assert (rows[2][0] == 3)
assert (rows[2][1] == "Carl")
assert (rows[2][2] == "*****@*****.**")
assert (rows[2][3] is None)
def test_is_primary_key(self):
self.connect(self.mysql)
table = Table("user",
self.adapter.metadata,
autoload=True,
autoload_with=self.adapter.engine)
columns = table.columns
assert (table.columns.has_key("userid") == True)
assert (self.adapter._is_primary_key(table.columns["userid"]) == True)
assert (table.columns.has_key("username") == True)
assert (self.adapter._is_primary_key(
table.columns["username"]) == False)
def test_get_primary_key(self):
self.connect(self.mysql)
user_table_primary_key = self.adapter.get_primary_key("user")
assert (type(user_table_primary_key) == type(""))
assert (user_table_primary_key == "userid")
def test_get_direction(self):
direction = self.adapter._get_direction("first")
assert (direction == asc)
direction = self.adapter._get_direction("last")
assert (direction == desc)
direction = self.adapter._get_direction("wrong")
assert (direction == None)
def test_get_column_index(self):
keys = [u'userid', u'username', u'email_address', u'date_created']
index = self.adapter._get_column_index(keys, "userid")
assert (index == 0)
index = self.adapter._get_column_index(keys, "email_address")
assert (index == 2)
index = self.adapter._get_column_index(keys, "wrong_field")
assert (index == None)
def test_get_last_inserted_id(self):
self.connect(self.mysql)
table = "user"
last_inserted_id = self.adapter.get_last_inserted_id(
table, "userid", "first")
assert (last_inserted_id == 1)
last_inserted_id = self.adapter.get_last_inserted_id(
table, "userid", "last")
assert (last_inserted_id == 22816)
# None, wrongcolumn is not primary key
last_inserted_id = self.adapter.get_last_inserted_id(
table, "username", "last")
assert (last_inserted_id == None)
last_inserted_id = self.adapter.get_last_inserted_id(
table, "wrongcolumn", "last")
assert (last_inserted_id == None)
def test_get_select_query(self):
columns = {u'first-name': u'username', u'email': u'email_address'}
select_query = self.adapter._get_select_query("user", columns,
"userid", 1)
assert (
select_query ==
"select username as \"first-name\", email_address as \"email\" from user where userid > 1"
)
columns = {u'email': u'email_address', u'name': u'username'}
select_query = self.adapter._get_select_query("table_name", columns,
"id", 991)
assert (
select_query ==
"select email_address as \"email\", username as \"name\" from table_name where id > 991"
)
def test_get_rows_to_be_inserted(self):
self.connect(self.mysql)
table = "user"
columns = {u'first-name': u'username', u'email': u'email_address'}
rows_to_be_inserted = self.adapter.get_rows_to_be_inserted(
table, columns, "userid", 1)
assert (len(rows_to_be_inserted) == 22815)
columns = {u'email': u'email_address', u'name': u'username'}
rows_to_be_inserted = self.adapter.get_rows_to_be_inserted(
table, columns, "userid", 10000)
assert (len(rows_to_be_inserted) == 12816)
columns = {u'email': u'email_address', u'name': u'username'}
rows_to_be_inserted = self.adapter.get_rows_to_be_inserted(
table, columns, "userid", 22816)
assert (rows_to_be_inserted == [])
class TestSync(object):
def setUp(self):
self.id = "5073d616dd0ef409aaaaeeec0fee"
self.mysql = {
"driver": "mysql",
"username": "******",
"password": "******",
"host": "localhost",
"port": 3066,
"database": "testdbfortests"
}
self.adapter = Adapter()
self.adapter.setup(self.mysql)
self.details = mongo_backend.get_collection("details")
self.detail = self.details.find_one({"_id": ObjectId(self.id)})
self.sync = Sync(self.id)
def test_split_rows(self):
chunks = self.sync.split_rows(range(1005), 10)
chunks_len = len(chunks)
assert (chunks_len == 101)
for index, chunk in enumerate(chunks):
if index < (chunks_len - 1):
assert (len(chunk) == 10)
else:
assert (len(chunk) == 5)
chunks = self.sync.split_rows(range(0), 10)
chunks_len = len(chunks)
assert (chunks_len == 0)
chunks = self.sync.split_rows(range(5), 10)
chunks_len = len(chunks)
assert (chunks_len == 1)
assert (len(chunks[0]) == 5)
def test_get_last_inserted_id(self):
last_inserted_id = self.sync.get_last_inserted_id(self.detail["list"])
assert (last_inserted_id == 22816)
last_inserted_id = self.sync.get_last_inserted_id({})
assert (last_inserted_id == None)
last_inserted_id = self.sync.get_last_inserted_id({"status": {}})
assert (last_inserted_id == None)
def test_get_last_id(self):
last_id = self.sync.get_last_id(self.adapter, "user", "userid",
self.detail["list"])
assert (last_id == 22816)
last_id = self.sync.get_last_id(self.adapter, "user", "userid", {})
assert (last_id == 1)
def test_update_status_running(self):
# setup
statebefore = self.detail
last_inserted_id = 1234
rows_to_be_inserted = 5678
assert (self.sync.update_status_running(last_inserted_id,
rows_to_be_inserted) == True)
detail = self.details.find_one({"_id": ObjectId(self.id)})
assert (detail["list"]["status"]["status"] == "Running")
assert (
detail["list"]["status"]["last_inserted_id"] == last_inserted_id)
assert (detail["list"]["status"]["rows_to_be_inserted"] ==
rows_to_be_inserted)
assert (detail["list"]["status"]["inserted_rows"] == 0)
# tear down
self.detail = statebefore
self.details.save(statebefore)
self.details.ensure_index("_id")
def test_update_status_completed(self):
# setup
statebefore = self.detail
last_id = 1234
assert (self.sync.update_status_completed(last_id) == True)
detail = self.details.find_one({"_id": ObjectId(self.id)})
assert (detail["list"]["status"]["status"] == "Completed")
assert (detail["list"]["status"]["last_inserted_id"] == last_id)
# tear down
self.detail = statebefore
self.details.save(statebefore)
self.details.ensure_index("_id")
def test_sync_users(self):
# setup
statebefore = self.detail
campaignmonitor = CampaignMonitor(
"787e70d714cdce3178610ddde2d7da08deeeeeddddd")
campaignmonitor_list_id = "fl8b40a4d15de4e7d83ead7e6b839544ib"
header_row = tuple(self.detail["columns"].keys())
user_chunks = [[('John', '*****@*****.**'),
('Susan', '*****@*****.**'),
('Carl', '*****@*****.**')]]
assert (self.sync.sync_users(campaignmonitor, campaignmonitor_list_id,
header_row, user_chunks, 22816) == True)
detail = self.details.find_one({"_id": ObjectId(self.id)})
assert (detail["list"]["status"]["inserted_rows"] ==
statebefore["list"]["status"]["inserted_rows"] + 3)
# teardown
self.details.save(statebefore)
self.details.ensure_index("_id")
def test_get_provider(self):
provider = self.sync.get_provider(self.detail["provider"], "apikey")
assert (provider is not None)
assert (isinstance(provider, CampaignMonitor))
# mocking provider data
provider_data = {}
provider_data["provider"] = "mailchimp"
provider = self.sync.get_provider(provider_data, "apikey")
assert (provider is not None)
assert (isinstance(provider, MailChimp))
def test_status(self):
list_data = self.detail["list"]
status = self.sync.status("Completed", list_data)
assert (status == True)
status = self.sync.status("Completed", {})
assert (status == False)
rdetail = self.details.find_one(
{"_id": ObjectId("5073d5f0dd0ef409aaec0fed")})
status = self.sync.status("Completed", rdetail["list"])
assert (status == False)
def test_sync_status_completed(self):
time = unix_utc_now()
status = {
"status": "completed",
"message": "Successfully synced.",
"progress": 100,
"last_synced": time
}
assert (self.sync.sync_status_completed(time) == status)
def test_sync_status_running(self):
inserted_rows = 1234
rows_to_be_inserted = 5678
status = {
"status": "running",
"message": "Mailsync is still syncing.",
"progress": (inserted_rows * 100) / rows_to_be_inserted,
"inserted_rows": inserted_rows,
"rows_to_be_inserted": rows_to_be_inserted
}
rows_to_be_inserted = 0
status = {
"status": "running",
"message": "Mailsync is still syncing.",
"progress": 0,
"inserted_rows": inserted_rows,
"rows_to_be_inserted": rows_to_be_inserted
}
assert (self.sync.sync_status_running(inserted_rows,
rows_to_be_inserted) == status)
def test_sync_status_error(self):
status = {
"status": "error",
"message": "An error occured, emails are not successfully synced."
}
assert (self.sync.sync_status_error() == status)
def test_sync_status_delete(self):
success_status = {
"status": "deleted",
"message": "Sync is successfully removed."
}
assert (self.sync.sync_status_delete("Success") == success_status)
error_status = {
"status": "error",
"message": "An error occured, sync is not successfully removed."
}
assert (self.sync.sync_status_delete("Error") == error_status)
assert (self.sync.sync_status_delete("Wrong") == error_status)
def test_delete(self):
# setup
statebefore = self.detail
self.sync._delete()
detail = self.details.find_one({"_id": ObjectId(self.id)})
assert (detail == None)
# teardown
self.details.save(statebefore)
self.details.ensure_index("_id")
def test_delete_sync(self):
# setup
statebefore = self.detail
result = self.sync.delete_sync()
detail = self.details.find_one({"_id": ObjectId(self.id)})
assert (detail == None)
success_status = {
"status": "deleted",
"message": "Sync is successfully removed."
}
assert (result == success_status)
# teardown
self.details.save(statebefore)
self.details.ensure_index("_id")
class TestAdapter(object):
def setUp(self):
self.adapter = Adapter()
self.mysql = {
"driver": "mysql",
"username": "******",
"password": "******",
"host": "localhost",
"port": 3066,
"database": "testdbfortests"
}
self.postgresql = {
"driver": "postgresql",
"username": "******",
"password": "******",
"host": "localhost",
"port": 5432,
"database": "testdb"
}
self.wrong = {
"driver": "wrongdriver",
"username": "******",
"password": "******",
"host": "localhost",
"port": 3066,
"database": "testdb"
}
def test_create_connection_string(self):
# mysql connection string
eq_(self.adapter.create_connection_string(self.mysql), "mysql://*****:*****@localhost:3066/testdbfortests")
# postgresql connection string
eq_(self.adapter.create_connection_string(self.postgresql), "postgresql://*****:*****@localhost:5432/testdb")
def test_get_engine_valid_parameters(self):
# mysql connection string
self.adapter.connection_string = self.adapter.create_connection_string(self.mysql)
assert self.adapter.get_engine() is not None
# postgresql connection string
self.adapter.connection_string = self.adapter.create_connection_string(self.postgresql)
assert self.adapter.get_engine() is not None
# wrong connection string
self.adapter.connection_string = self.adapter.create_connection_string(self.wrong)
assert self.adapter.get_engine() is None
def test_get_engine_metadata(self):
# mysql connection string
self.adapter.connection_string = self.adapter.create_connection_string(self.mysql)
self.adapter.engine = self.adapter.get_engine()
assert self.adapter.get_engine_metadata(self.adapter.engine, "reflect=True") is not None
# wrong connection string
self.adapter.connection_string = self.adapter.create_connection_string(self.wrong)
self.adapter.engine = self.adapter.get_engine()
assert self.adapter.get_engine_metadata(self.adapter.engine, "reflect=True") is None
def test_is_valid_connection(self):
# mysql connection string
self.connect(self.mysql)
assert self.adapter.engine and self.adapter.metadata
# postgresql connection string
self.connect(self.postgresql)
assert self.adapter.engine and self.adapter.metadata
# wrong connection string
self.connect(self.wrong)
assert not (self.adapter.engine and self.adapter.metadata)
def connect(self, connection_string):
self.adapter.connection_string = self.adapter.create_connection_string(connection_string)
self.adapter.engine = self.adapter.get_engine()
self.adapter.metadata = self.adapter.get_engine_metadata(self.adapter.engine, "reflect=True")
def test_get_tables(self):
self.connect(self.mysql)
tables = self.adapter.get_tables()
assert(tables is not None)
assert(len(tables) == 1)
assert(tables[0] == "user")
def test_get_columns(self):
self.connect(self.mysql)
columns = self.adapter.get_columns("user")
assert(columns is not None)
assert(len(columns) == 4)
assert(columns[0] == "userid")
assert(columns[1] == "username")
assert(columns[2] == "email_address")
assert(columns[3] == "date_created")
def test_get_rows(self):
# mysql connection string
self.connect(self.mysql)
rows = self.adapter.get_top_rows("user")
assert(rows is not None)
assert(len(rows) == 3)
assert(len(rows[0]) == 4)
assert(len(rows[1]) == 4)
assert(len(rows[2]) == 4)
assert(rows[0][0] == 1)
assert(rows[0][1] == "John")
assert(rows[0][2] == "*****@*****.**")
assert(rows[0][3] is None)
assert(rows[1][0] == 2)
assert(rows[1][1] == "Susan")
assert(rows[1][2] == "*****@*****.**")
assert(rows[1][3] is None)
assert(rows[2][0] == 3)
assert(rows[2][1] == "Carl")
assert(rows[2][2] == "*****@*****.**")
assert(rows[2][3] is None)
def test_is_primary_key(self):
self.connect(self.mysql)
table = Table("user", self.adapter.metadata, autoload=True, autoload_with=self.adapter.engine)
columns = table.columns
assert(table.columns.has_key("userid") == True)
assert(self.adapter._is_primary_key(table.columns["userid"]) == True)
assert(table.columns.has_key("username") == True)
assert(self.adapter._is_primary_key(table.columns["username"]) == False)
def test_get_primary_key(self):
self.connect(self.mysql)
user_table_primary_key = self.adapter.get_primary_key("user")
assert(type(user_table_primary_key) == type(""))
assert(user_table_primary_key == "userid")
def test_get_direction(self):
direction = self.adapter._get_direction("first")
assert(direction == asc)
direction = self.adapter._get_direction("last")
assert(direction == desc)
direction = self.adapter._get_direction("wrong")
assert(direction == None)
def test_get_column_index(self):
keys = [u'userid', u'username', u'email_address', u'date_created']
index = self.adapter._get_column_index(keys, "userid")
assert(index == 0)
index = self.adapter._get_column_index(keys, "email_address")
assert(index == 2)
index = self.adapter._get_column_index(keys, "wrong_field")
assert(index == None)
def test_get_last_inserted_id(self):
self.connect(self.mysql)
table = "user"
last_inserted_id = self.adapter.get_last_inserted_id(table, "userid", "first")
assert(last_inserted_id == 1)
last_inserted_id = self.adapter.get_last_inserted_id(table, "userid", "last")
assert(last_inserted_id == 22816)
# None, wrongcolumn is not primary key
last_inserted_id = self.adapter.get_last_inserted_id(table, "username", "last")
assert(last_inserted_id == None)
last_inserted_id = self.adapter.get_last_inserted_id(table, "wrongcolumn", "last")
assert(last_inserted_id == None)
def test_get_select_query(self):
columns = {u'first-name': u'username', u'email': u'email_address'}
select_query = self.adapter._get_select_query("user", columns, "userid", 1)
assert(select_query == "select username as \"first-name\", email_address as \"email\" from user where userid > 1")
columns = {u'email': u'email_address', u'name': u'username'}
select_query = self.adapter._get_select_query("table_name", columns, "id", 991)
assert(select_query == "select email_address as \"email\", username as \"name\" from table_name where id > 991")
def test_get_rows_to_be_inserted(self):
self.connect(self.mysql)
table = "user"
columns = {u'first-name': u'username', u'email': u'email_address'}
rows_to_be_inserted = self.adapter.get_rows_to_be_inserted(table, columns, "userid", 1)
assert(len(rows_to_be_inserted) == 22815)
columns = {u'email': u'email_address', u'name': u'username'}
rows_to_be_inserted = self.adapter.get_rows_to_be_inserted(table, columns, "userid", 10000)
assert(len(rows_to_be_inserted) == 12816)
columns = {u'email': u'email_address', u'name': u'username'}
rows_to_be_inserted = self.adapter.get_rows_to_be_inserted(table, columns, "userid", 22816)
assert(rows_to_be_inserted == [])
def __init__(self, connection_details):
self.connection_details = connection_details
self.adapter = Adapter()
self.adapter.setup(self.connection_details)
def __init__(self, connection_details): self.connection_details = connection_details self.adapter = Adapter() self.adapter.setup(self.connection_details)