def test_no_error(self, mocked_logger, mocked_connect): mocked_connect.side_effect = paramiko.SFTPClient mocked_connect.open.side_effect = mock.mock_open() conn = client.SFTPConnection("10.0.0.1", "username", port="22") file_handle = conn.get_file_handle({ "filepath": "/root_dir/file.csv.gz", "last_modified": "2020-01-01" }) self.assertEquals(0, mocked_logger.call_count)
def test_sorted_files(self, mocked_all_files): conn = client.SFTPConnection("10.0.0.1", "username", port="22") files_list = [{ "filepath": "/root/file1.csv", "last_modified": datetime.utcfromtimestamp(time.time() - 10).replace(tzinfo=pytz.UTC) }, { "filepath": "/root/file2.csv", "last_modified": datetime.utcfromtimestamp(time.time() - 4).replace(tzinfo=pytz.UTC) }, { "filepath": "/root/file3.csv", "last_modified": datetime.utcfromtimestamp(time.time() - 8).replace(tzinfo=pytz.UTC) }, { "filepath": "/root/file4.csv", "last_modified": datetime.utcfromtimestamp(time.time()).replace(tzinfo=pytz.UTC) }, { "filepath": "/root/file.txt", "last_modified": datetime.utcfromtimestamp(time.time() - 2).replace(tzinfo=pytz.UTC) }, { "filepath": "/root/file5.json", "last_modified": datetime.utcfromtimestamp(time.time() - 3).replace(tzinfo=pytz.UTC) }] mocked_all_files.return_value = files_list files = conn.get_files("/root", "file[0-9].csv") # expected files in increasing order of "last_modified" expected_files_list = [ "/root/file1.csv", "/root/file3.csv", "/root/file2.csv", "/root/file4.csv" ] actual_files_list = [f["filepath"] for f in files] self.assertEquals(expected_files_list, actual_files_list)
def test_no_error_during_sync(self, mocked_get_row_iterators, mocked_stats, mocked_logger, mocked_connect): mocked_connect.side_effect = paramiko.SFTPClient mocked_get_row_iterators.return_value = [] mocked_connect.open.side_effect = mock.mock_open() conn = client.SFTPConnection("10.0.0.1", "username", port="22") rows_synced = sync.sync_file(conn, { "filepath": "/root_dir/file.csv.gz", "last_modified": "2020-01-01" }, None, { "key_properties": ["id"], "delimiter": "," }) # check if "csv.get_row_iterators" is called if it is called then error has not occurred # if it is not called then error has occured and function returned from the except block self.assertEquals(1, mocked_get_row_iterators.call_count)
def test_permission_error(self, mocked_logger, mocked_connect): mocked_connect.side_effect = paramiko.SFTPClient mocked_connect.open.side_effect = PermissionError("Permission denied") conn = client.SFTPConnection("10.0.0.1", "username", port="22") try: file_handle = conn.get_file_handle({ "filepath": "/root_dir/file.csv.gz", "last_modified": "2020-01-01" }) except OSError: # check if logger is called if logger is called in the function # then error has occurred otherwise not mocked_logger.assert_called_with( "Skipping %s file because you do not have enough permissions.", "/root_dir/file.csv.gz")
def test_permisison_error_during_sync(self, mocked_get_row_iterators, mocked_logger, mocked_connect): mocked_connect.side_effect = paramiko.SFTPClient mocked_get_row_iterators.return_value = [] mocked_connect.open.side_effect = PermissionError("Permission denied") conn = client.SFTPConnection("10.0.0.1", "username", port="22") rows_synced = sync.sync_file(conn, { "filepath": "/root_dir/file.csv.gz", "last_modified": "2020-01-01" }, None, { "key_properties": ["id"], "delimiter": "," }) # check if "csv.get_row_iterators" is called if it is called then error has not occurred # if it is not called then error has occured and function returned from the except block self.assertEquals(0, mocked_get_row_iterators.call_count) mocked_logger.assert_called_with( "Skipping %s file because you do not have enough permissions.", "/root_dir/file.csv.gz")
def test_sorted_files_negative(self, mocked_all_files): conn = client.SFTPConnection("10.0.0.1", "username", port="22") files_list = [{ "filepath": "/root/file1.csv", "last_modified": datetime.utcfromtimestamp(time.time() - 3).replace(tzinfo=pytz.UTC) }, { "filepath": "/root/file2.csv", "last_modified": datetime.utcfromtimestamp(time.time()).replace(tzinfo=pytz.UTC) }, { "filepath": "/root/file.txt", "last_modified": datetime.utcfromtimestamp(time.time() - 2).replace(tzinfo=pytz.UTC) }, { "filepath": "/root/file3.json", "last_modified": datetime.utcfromtimestamp(time.time() - 5).replace(tzinfo=pytz.UTC) }] mocked_all_files.return_value = files_list # setting "modified_since" to now modified_since = singer.utils.strptime_to_utc( datetime.utcnow().replace(tzinfo=pytz.UTC).isoformat()) files = conn.get_files("/root", "file[0-9].csv", modified_since) # as all the modified date is lesser than "modified_since" thus, no files will be returned expected_files_list = [] actual_files_list = [f["filepath"] for f in files] self.assertEquals(expected_files_list, actual_files_list)