def test_get_directory(mocker, mock_client): dirname = "local-path/" os_path_dirname_mock = mocker.patch("os.path.dirname", return_value=dirname) os_path_isdir_mock = mocker.patch("os.path.isdir", return_value=True) local_dests = ["local-path/project-path", "local-path/project-path/file"] os_path_join_mock = mocker.patch("os.path.join", side_effect=local_dests) os_path_exists_mock = mocker.patch("os.path.exists", return_value=False) os_makedirs_mock = mocker.patch("os.makedirs", return_value=None) relative_path1 = mocker.Mock() relative_path2 = mocker.Mock() relative_paths = [relative_path1, relative_path2] get_relative_path_mock = mocker.patch( "faculty.datasets.util.get_relative_path", side_effect=relative_paths) ls_mock = mocker.patch( "faculty.datasets.ls", return_value=["/project-path/", "/project-path/test-file"], ) _get_file_mock = mocker.patch("faculty.datasets._get_file", return_value=None) datasets.get("project-path", "local-path", PROJECT_ID) os_path_dirname_mock.assert_has_calls( [mocker.call("local-path"), mocker.call(local_dests[1])]) os_path_isdir_mock.assert_called_once_with("local-path/") get_relative_path_mock.assert_has_calls([ mocker.call("project-path", "/project-path/"), mocker.call("project-path", "/project-path/test-file"), ]) os_path_join_mock.assert_has_calls([ mocker.call("local-path", relative_path1), mocker.call("local-path", relative_path2), ]) os_path_exists_mock.assert_has_calls( [mocker.call(local_dests[0]), mocker.call(dirname)]) os_makedirs_mock.assert_has_calls( [mocker.call(local_dests[0]), mocker.call(dirname)]) ls_mock.assert_has_calls([ mocker.call( "project-path/", project_id=PROJECT_ID, show_hidden=True, object_client=mock_client, ), mocker.call( "project-path", project_id=PROJECT_ID, show_hidden=True, object_client=mock_client, ), ]) _get_file_mock.assert_called_once_with("/project-path/test-file", local_dests[1], PROJECT_ID, mock_client)
def get_todays_dataframe(): if TODAY_CSV in datasets.ls(): datasets.get(TODAY_CSV, TMP_CSV) df = pd.read_csv(TMP_CSV, sep='\t', encoding='utf-8') if os.path.exists(TMP_CSV): os.remove(TMP_CSV) return df return None
def test_get_file(mocker, mock_client): ls_mock = mocker.patch("faculty.datasets.ls", return_value=[]) download_mock = mocker.patch("faculty.datasets.transfer.download_file") datasets.get("project-path", "local-path", PROJECT_ID) ls_mock.assert_called_once_with( "project-path/", project_id=PROJECT_ID, show_hidden=True, object_client=mock_client, ) download_mock.assert_called_once_with(mock_client, PROJECT_ID, "project-path", "local-path")
def test_get_empty_directory(mocker, mock_client): dirname = "local-path/" os_path_dirname_mock = mocker.patch( "os.path.dirname", return_value=dirname ) os_path_isdir_mock = mocker.patch("os.path.isdir", return_value=True) local_dest = "local-path/test-path" os_path_join_mock = mocker.patch("os.path.join", return_value=local_dest) os_path_exists_mock = mocker.patch("os.path.exists", return_value=False) os_makedirs_mock = mocker.patch("os.makedirs", return_value=None) relative_path = mocker.Mock() get_relative_path_mock = mocker.patch( "faculty.datasets.util.get_relative_path", return_value=relative_path ) ls_mock = mocker.patch( "faculty.datasets.ls", return_value=["/project-path/"] ) datasets.get("project-path", "local-path", PROJECT_ID) os_path_dirname_mock.assert_called_once_with("local-path") os_path_isdir_mock.assert_called_once_with("local-path/") get_relative_path_mock.assert_called_once_with( "project-path", "/project-path/" ) os_path_join_mock.assert_called_once_with("local-path", relative_path) os_path_exists_mock.assert_called_once_with(local_dest) os_makedirs_mock.assert_called_once_with(local_dest) ls_mock.assert_has_calls( [ mocker.call( "project-path/", project_id=PROJECT_ID, show_hidden=True, object_client=mock_client, ), mocker.call( "project-path", project_id=PROJECT_ID, show_hidden=True, object_client=mock_client, ), ] )
def _download_file(self, remote_file_path, local_path): datasets_path = self._datasets_path(remote_file_path) datasets.get(datasets_path, local_path, self.project_id)