def test_get_directory(mocker, mock_client):
dirname = "local-path/"
os_path_dirname_mock = mocker.patch("os.path.dirname",
return_value=dirname)
os_path_isdir_mock = mocker.patch("os.path.isdir", return_value=True)
local_dests = ["local-path/project-path", "local-path/project-path/file"]
os_path_join_mock = mocker.patch("os.path.join", side_effect=local_dests)
os_path_exists_mock = mocker.patch("os.path.exists", return_value=False)
os_makedirs_mock = mocker.patch("os.makedirs", return_value=None)
relative_path1 = mocker.Mock()
relative_path2 = mocker.Mock()
relative_paths = [relative_path1, relative_path2]
get_relative_path_mock = mocker.patch(
"faculty.datasets.util.get_relative_path", side_effect=relative_paths)
ls_mock = mocker.patch(
"faculty.datasets.ls",
return_value=["/project-path/", "/project-path/test-file"],
)
_get_file_mock = mocker.patch("faculty.datasets._get_file",
return_value=None)
datasets.get("project-path", "local-path", PROJECT_ID)
os_path_dirname_mock.assert_has_calls(
[mocker.call("local-path"),
mocker.call(local_dests[1])])
os_path_isdir_mock.assert_called_once_with("local-path/")
get_relative_path_mock.assert_has_calls([
mocker.call("project-path", "/project-path/"),
mocker.call("project-path", "/project-path/test-file"),
])
os_path_join_mock.assert_has_calls([
mocker.call("local-path", relative_path1),
mocker.call("local-path", relative_path2),
])
os_path_exists_mock.assert_has_calls(
[mocker.call(local_dests[0]),
mocker.call(dirname)])
os_makedirs_mock.assert_has_calls(
[mocker.call(local_dests[0]),
mocker.call(dirname)])
ls_mock.assert_has_calls([
mocker.call(
"project-path/",
project_id=PROJECT_ID,
show_hidden=True,
object_client=mock_client,
),
mocker.call(
"project-path",
project_id=PROJECT_ID,
show_hidden=True,
object_client=mock_client,
),
])
_get_file_mock.assert_called_once_with("/project-path/test-file",
local_dests[1], PROJECT_ID,
mock_client)
def get_todays_dataframe():
if TODAY_CSV in datasets.ls():
datasets.get(TODAY_CSV, TMP_CSV)
df = pd.read_csv(TMP_CSV, sep='\t', encoding='utf-8')
if os.path.exists(TMP_CSV):
os.remove(TMP_CSV)
return df
return None
def test_get_file(mocker, mock_client):
ls_mock = mocker.patch("faculty.datasets.ls", return_value=[])
download_mock = mocker.patch("faculty.datasets.transfer.download_file")
datasets.get("project-path", "local-path", PROJECT_ID)
ls_mock.assert_called_once_with(
"project-path/",
project_id=PROJECT_ID,
show_hidden=True,
object_client=mock_client,
)
download_mock.assert_called_once_with(mock_client, PROJECT_ID,
"project-path", "local-path")
def test_get_empty_directory(mocker, mock_client):
dirname = "local-path/"
os_path_dirname_mock = mocker.patch(
"os.path.dirname", return_value=dirname
)
os_path_isdir_mock = mocker.patch("os.path.isdir", return_value=True)
local_dest = "local-path/test-path"
os_path_join_mock = mocker.patch("os.path.join", return_value=local_dest)
os_path_exists_mock = mocker.patch("os.path.exists", return_value=False)
os_makedirs_mock = mocker.patch("os.makedirs", return_value=None)
relative_path = mocker.Mock()
get_relative_path_mock = mocker.patch(
"faculty.datasets.util.get_relative_path", return_value=relative_path
)
ls_mock = mocker.patch(
"faculty.datasets.ls", return_value=["/project-path/"]
)
datasets.get("project-path", "local-path", PROJECT_ID)
os_path_dirname_mock.assert_called_once_with("local-path")
os_path_isdir_mock.assert_called_once_with("local-path/")
get_relative_path_mock.assert_called_once_with(
"project-path", "/project-path/"
)
os_path_join_mock.assert_called_once_with("local-path", relative_path)
os_path_exists_mock.assert_called_once_with(local_dest)
os_makedirs_mock.assert_called_once_with(local_dest)
ls_mock.assert_has_calls(
[
mocker.call(
"project-path/",
project_id=PROJECT_ID,
show_hidden=True,
object_client=mock_client,
),
mocker.call(
"project-path",
project_id=PROJECT_ID,
show_hidden=True,
object_client=mock_client,
),
]
)
def _download_file(self, remote_file_path, local_path):
datasets_path = self._datasets_path(remote_file_path)
datasets.get(datasets_path, local_path, self.project_id)