def get_todays_dataframe():
if TODAY_CSV in datasets.ls():
datasets.get(TODAY_CSV, TMP_CSV)
df = pd.read_csv(TMP_CSV, sep='\t', encoding='utf-8')
if os.path.exists(TMP_CSV):
os.remove(TMP_CSV)
return df
return None
def test_ls_files_hide_hidden_files(mocker, mock_client):
list_response = mocker.Mock()
list_response.objects = [
mocker.Mock(path=".test-hidden"),
mocker.Mock(path="not-hidden"),
]
list_response.next_page_token = None
mock_client.list.return_value = list_response
objects = datasets.ls("test-prefix", PROJECT_ID)
assert objects == ["not-hidden"]
mock_client.list.assert_called_once_with(PROJECT_ID, "test-prefix")
def test_ls_with_continuation(mocker, mock_client):
list_response1 = mocker.Mock()
mock_object1 = mocker.Mock()
mock_object1.path = ".test-hidden-path"
list_response1.objects = [mock_object1]
list_response2 = mocker.Mock()
mock_object2 = mocker.Mock()
mock_object2.path = "test-path"
list_response2.objects = [mock_object2]
list_response2.next_page_token = None
mock_client.list.side_effect = [list_response1, list_response2]
objects = datasets.ls("test-prefix", PROJECT_ID)
assert objects == ["test-path"]
mock_client.list.assert_has_calls([
mocker.call(PROJECT_ID, "test-prefix"),
mocker.call(PROJECT_ID, "test-prefix", list_response1.next_page_token),
])
