def test_no_error(self, mocked_logger, mocked_connect):
mocked_connect.side_effect = paramiko.SFTPClient
mocked_connect.open.side_effect = mock.mock_open()
conn = client.SFTPConnection("10.0.0.1", "username", port="22")
file_handle = conn.get_file_handle({
"filepath": "/root_dir/file.csv.gz",
"last_modified": "2020-01-01"
})
self.assertEquals(0, mocked_logger.call_count)
def test_sorted_files(self, mocked_all_files):
conn = client.SFTPConnection("10.0.0.1", "username", port="22")
files_list = [{
"filepath":
"/root/file1.csv",
"last_modified":
datetime.utcfromtimestamp(time.time() -
10).replace(tzinfo=pytz.UTC)
}, {
"filepath":
"/root/file2.csv",
"last_modified":
datetime.utcfromtimestamp(time.time() - 4).replace(tzinfo=pytz.UTC)
}, {
"filepath":
"/root/file3.csv",
"last_modified":
datetime.utcfromtimestamp(time.time() - 8).replace(tzinfo=pytz.UTC)
}, {
"filepath":
"/root/file4.csv",
"last_modified":
datetime.utcfromtimestamp(time.time()).replace(tzinfo=pytz.UTC)
}, {
"filepath":
"/root/file.txt",
"last_modified":
datetime.utcfromtimestamp(time.time() - 2).replace(tzinfo=pytz.UTC)
}, {
"filepath":
"/root/file5.json",
"last_modified":
datetime.utcfromtimestamp(time.time() - 3).replace(tzinfo=pytz.UTC)
}]
mocked_all_files.return_value = files_list
files = conn.get_files("/root", "file[0-9].csv")
# expected files in increasing order of "last_modified"
expected_files_list = [
"/root/file1.csv", "/root/file3.csv", "/root/file2.csv",
"/root/file4.csv"
]
actual_files_list = [f["filepath"] for f in files]
self.assertEquals(expected_files_list, actual_files_list)
def test_no_error_during_sync(self, mocked_get_row_iterators, mocked_stats,
mocked_logger, mocked_connect):
mocked_connect.side_effect = paramiko.SFTPClient
mocked_get_row_iterators.return_value = []
mocked_connect.open.side_effect = mock.mock_open()
conn = client.SFTPConnection("10.0.0.1", "username", port="22")
rows_synced = sync.sync_file(conn, {
"filepath": "/root_dir/file.csv.gz",
"last_modified": "2020-01-01"
}, None, {
"key_properties": ["id"],
"delimiter": ","
})
# check if "csv.get_row_iterators" is called if it is called then error has not occurred
# if it is not called then error has occured and function returned from the except block
self.assertEquals(1, mocked_get_row_iterators.call_count)
def test_permission_error(self, mocked_logger, mocked_connect):
mocked_connect.side_effect = paramiko.SFTPClient
mocked_connect.open.side_effect = PermissionError("Permission denied")
conn = client.SFTPConnection("10.0.0.1", "username", port="22")
try:
file_handle = conn.get_file_handle({
"filepath": "/root_dir/file.csv.gz",
"last_modified": "2020-01-01"
})
except OSError:
# check if logger is called if logger is called in the function
# then error has occurred otherwise not
mocked_logger.assert_called_with(
"Skipping %s file because you do not have enough permissions.",
"/root_dir/file.csv.gz")
def test_permisison_error_during_sync(self, mocked_get_row_iterators,
mocked_logger, mocked_connect):
mocked_connect.side_effect = paramiko.SFTPClient
mocked_get_row_iterators.return_value = []
mocked_connect.open.side_effect = PermissionError("Permission denied")
conn = client.SFTPConnection("10.0.0.1", "username", port="22")
rows_synced = sync.sync_file(conn, {
"filepath": "/root_dir/file.csv.gz",
"last_modified": "2020-01-01"
}, None, {
"key_properties": ["id"],
"delimiter": ","
})
# check if "csv.get_row_iterators" is called if it is called then error has not occurred
# if it is not called then error has occured and function returned from the except block
self.assertEquals(0, mocked_get_row_iterators.call_count)
mocked_logger.assert_called_with(
"Skipping %s file because you do not have enough permissions.",
"/root_dir/file.csv.gz")
def test_sorted_files_negative(self, mocked_all_files):
conn = client.SFTPConnection("10.0.0.1", "username", port="22")
files_list = [{
"filepath":
"/root/file1.csv",
"last_modified":
datetime.utcfromtimestamp(time.time() - 3).replace(tzinfo=pytz.UTC)
}, {
"filepath":
"/root/file2.csv",
"last_modified":
datetime.utcfromtimestamp(time.time()).replace(tzinfo=pytz.UTC)
}, {
"filepath":
"/root/file.txt",
"last_modified":
datetime.utcfromtimestamp(time.time() - 2).replace(tzinfo=pytz.UTC)
}, {
"filepath":
"/root/file3.json",
"last_modified":
datetime.utcfromtimestamp(time.time() - 5).replace(tzinfo=pytz.UTC)
}]
mocked_all_files.return_value = files_list
# setting "modified_since" to now
modified_since = singer.utils.strptime_to_utc(
datetime.utcnow().replace(tzinfo=pytz.UTC).isoformat())
files = conn.get_files("/root", "file[0-9].csv", modified_since)
# as all the modified date is lesser than "modified_since" thus, no files will be returned
expected_files_list = []
actual_files_list = [f["filepath"] for f in files]
self.assertEquals(expected_files_list, actual_files_list)