def test_credentials_create_from_credentials(self):
credentials1 = Credentials(api_key='abc123', username='******')
credentials2 = Credentials.from_credentials(credentials1)
assert credentials1.api_key == credentials2.api_key
assert credentials1.username == credentials2.username
assert credentials1.base_url == credentials2.base_url
def test_geography_download_without_do_enabled(self, mock_download_stream,
mock_get_by_id):
# Given
mock_get_by_id.return_value = test_geography1
geography = Geography.get(test_geography1.id)
def raise_exception(limit=None,
order_by=None,
sql_query=None,
add_geom=None,
is_geography=None):
raise ServerErrorException(
['The user does not have Data Observatory enabled'])
mock_download_stream.side_effect = raise_exception
credentials = Credentials('fake_user', '1234')
# When
with pytest.raises(Exception) as e:
geography.to_csv('fake_path', credentials)
# Then
assert str(e.value) == (
'We are sorry, the Data Observatory is not enabled for your account yet. '
'Please contact your customer success manager or send an email to '
'[email protected] to request access to it.')
def test_geography_subscription_info(self, mock_fetch):
# Given
mock_fetch.return_value = test_subscription_info
credentials = Credentials('fake_user', '1234')
geography = Geography(db_geography1)
# When
info = geography.subscription_info(credentials)
# Then
mock_fetch.assert_called_once_with(db_geography1['id'], 'geography',
credentials)
assert isinstance(info, SubscriptionInfo)
assert info.id == test_subscription_info['id']
assert info.estimated_delivery_days == test_subscription_info[
'estimated_delivery_days']
assert info.subscription_list_price == test_subscription_info[
'subscription_list_price']
assert info.tos == test_subscription_info['tos']
assert info.tos_link == test_subscription_info['tos_link']
assert info.licenses == test_subscription_info['licenses']
assert info.licenses_link == test_subscription_info['licenses_link']
assert info.rights == test_subscription_info['rights']
assert str(info) == 'Properties: id, estimated_delivery_days, ' + \
'subscription_list_price, tos, tos_link, ' + \
'licenses, licenses_link, rights'
def test_prepare_variables(self, get_mock, _validate_bq_operations_mock):
_validate_bq_operations_mock.return_value = True
variable_id = 'project.dataset.table.variable'
variable = Variable({
'id': variable_id,
'column_name': 'column',
'dataset_id': 'fake_name'
})
get_mock.return_value = variable
credentials = Credentials('fake_user', '1234')
one_variable_cases = [
variable_id,
variable
]
for case in one_variable_cases:
result = prepare_variables(case, credentials)
assert result == [variable]
several_variables_cases = [
[variable_id, variable_id],
[variable, variable],
[variable, variable_id]
]
for case in several_variables_cases:
result = prepare_variables(case, credentials)
assert result == [variable, variable]
def test_prepare_variables_raises_if_not_available_in_bq(self, get_mock, entity_repo, get_all_mock):
dataset = Dataset({
'id': 'id',
'slug': 'slug',
'name': 'name',
'description': 'description',
'available_in': [],
'geography_id': 'geography',
'is_public_data': False
})
# mock dataset
entity_repo.return_value = dataset
# mock subscriptions
get_all_mock.return_value = [dataset]
variable = Variable({
'id': 'id',
'column_name': 'column',
'dataset_id': 'fake_name',
'slug': 'slug'
})
get_mock.return_value = variable
credentials = Credentials('fake_user', '1234')
with pytest.raises(EnrichmentError) as e:
prepare_variables(variable, credentials)
error = """
The Dataset '{}' is not ready for Enrichment. Please, contact us for more information.
""".format(dataset)
assert str(e.value) == error
def test_prepare_variables_works_with_private_and_subscribed(self, get_mock, entity_repo, get_all_mock):
dataset = Dataset({
'id': 'id',
'slug': 'slug',
'name': 'name',
'description': 'description',
'available_in': ['bq'],
'geography_id': 'geography',
'is_public_data': False
})
# mock dataset
entity_repo.return_value = dataset
# mock subscriptions
get_all_mock.return_value = [dataset]
variable = Variable({
'id': 'id',
'column_name': 'column',
'dataset_id': 'fake_name',
'slug': 'slug'
})
get_mock.return_value = variable
credentials = Credentials('fake_user', '1234')
result = prepare_variables(variable, credentials)
assert result == [variable]
def test_prepare_variables_fails_with_private(self, get_mock, entity_repo, get_all_mock):
dataset = Dataset({
'id': 'id',
'slug': 'slug',
'name': 'name',
'description': 'description',
'available_in': ['bq'],
'geography_id': 'geography',
'is_public_data': False
})
# mock dataset
entity_repo.return_value = dataset
# mock subscriptions
get_all_mock.return_value = []
variable = Variable({
'id': 'id',
'column_name': 'column',
'dataset_id': 'fake_name',
'slug': 'slug'
})
get_mock.return_value = variable
credentials = Credentials('fake_user', '1234')
with pytest.raises(EnrichmentError) as e:
prepare_variables(variable, credentials)
error = """
You are not subscribed to the Dataset '{}' yet. Please, use the subscribe method first.
""".format(dataset.id)
assert str(e.value) == error
def test_prepare_variables_without_agg_method_and_custom_agg(self, get_mock, _validate_bq_operations_mock):
_validate_bq_operations_mock.return_value = True
variable_id = 'project.dataset.table.variable'
variable = Variable({
'id': variable_id,
'column_name': 'column',
'dataset_id': 'fake_name',
'agg_method': None
})
get_mock.return_value = variable
credentials = Credentials('fake_user', '1234')
one_variable_cases = [
variable_id,
variable
]
for case in one_variable_cases:
result = prepare_variables(case, credentials)
assert result == [variable]
for case in one_variable_cases:
result = prepare_variables(case, credentials, aggregation={})
assert result == []
for case in one_variable_cases:
result = prepare_variables(case, credentials, aggregation={variable_id: 'SUM'})
assert result == [variable]
def setup_method(self):
self.original_init_clients = BigQueryClient._init_clients
BigQueryClient._init_clients = Mock(return_value=(True, True, True))
self.username = '******'
self.apikey = 'apikey'
self.credentials = Credentials(self.username, self.apikey)
self.file_path = '/tmp/test_download.csv'
def setup_method(self):
if (os.environ.get('APIKEY') and os.environ.get('USERNAME')):
self.apikey = os.environ['APIKEY']
self.username = os.environ['USERNAME']
else:
creds = json.loads(open('tests/e2e/secret.json').read())
self.apikey = creds['APIKEY']
self.username = creds['USERNAME']
self.credentials = Credentials(self.username, self.apikey)
self.enrichment = Enrichment(self.credentials)
self.points_gdf = read_file(file_path('files/points.geojson'))
self.polygons_gdf = read_file(file_path('files/polygon.geojson'))
# from carto-do-public-data.usa_acs.demographics_sociodemographics_usa_censustract_2015_5yrs_20132017
self.public_variable1 = public_variable1
self.public_variable2 = public_variable2
self.public_variable3 = public_variable3
self.public_variables = [
self.public_variable1, self.public_variable2, self.public_variable3
]
# from carto-do.ags.demographics_retailpotential_usa_blockgroup_2015_yearly_2019
self.private_variable1 = private_variable1
self.private_variable2 = private_variable2
self.private_variable3 = private_variable3
self.private_variables = [
self.private_variable1, self.private_variable2,
self.private_variable3
]
def setup_method(self):
self.original_bigquery_Client = bigquery.Client
bigquery.Client = Mock(return_value=True)
self.original_storage_Client = storage.Client
storage.Client = Mock(return_value=True)
self.original_get_do_credentials = Credentials.get_do_credentials
Credentials.get_do_credentials = Mock(return_value=DoCredentials(_PUBLIC_PROJECT, _WORKING_PROJECT))
self.credentials = Credentials('username', 'apikey')
def setUp(self):
if os.environ.get('APIKEY') and os.environ.get(
'USERNAME') and os.environ.get('USERURL'):
self.apikey = os.environ['APIKEY']
self.username = os.environ['USERNAME']
self.base_url = os.environ['USERURL']
else:
creds = json.loads(open('tests/e2e/secret.json').read())
self.apikey = creds['APIKEY']
self.username = creds['USERNAME']
self.base_url = creds['USERURL']
credentials = Credentials(username=self.username,
api_key=self.apikey,
base_url=self.base_url)
auth_client = credentials.get_api_key_auth_client()
self.do_dataset = DODataset(auth_client=auth_client)
def test_credentials_repr(self):
credentials = Credentials(self.username, self.api_key)
ans = ("Credentials(username='******', "
"api_key='{api_key}', "
"base_url='https://{username}.carto.com')").format(username=self.username,
api_key=self.api_key)
assert str(credentials) == ans
def test_dataset_download_not_subscribed_but_public(self, mocked_bq_client, get_by_id_mock, get_all_mock):
# Given
get_by_id_mock.return_value = test_dataset1 # is public
dataset = Dataset.get(test_dataset1.id)
get_all_mock.return_value = []
mocked_bq_client.return_value = BigQueryClientMock()
credentials = Credentials('fake_user', '1234')
dataset.to_csv('fake_path', credentials)
def test_geography_download_not_subscribed_but_public(
self, download_stream_mock, get_by_id_mock, get_all_mock):
# Given
get_by_id_mock.return_value = test_geography1 # is public
geography = Geography.get(test_geography1.id)
get_all_mock.return_value = []
download_stream_mock.return_value = []
credentials = Credentials('fake_user', '1234')
geography.to_csv('fake_path', credentials)
def test_dataset_download(self, mocked_bq_client, get_by_id_mock, get_all_mock):
# Given
get_by_id_mock.return_value = test_dataset1
dataset = Dataset.get(test_dataset1.id)
get_all_mock.return_value = [dataset]
mocked_bq_client.return_value = BigQueryClientMock()
credentials = Credentials('fake_user', '1234')
# Then
dataset.to_csv('fake_path', credentials)
def test_dataset_download_not_subscribed_but_public(
self, download_stream_mock, get_by_id_mock, get_all_mock):
# Given
get_by_id_mock.return_value = test_dataset1 # is public
dataset = Dataset.get(test_dataset1.id)
get_all_mock.return_value = []
download_stream_mock.return_value = []
credentials = Credentials('fake_user', '1234')
dataset.to_csv('fake_path', credentials)
def test_geography_download_not_subscribed_but_public(
self, mock_download_stream, mock_get_by_id):
# Given
mock_get_by_id.return_value = test_geography1 # is public
geography = Geography.get(test_geography1.id)
mock_download_stream.return_value = []
credentials = Credentials('fake_user', '1234')
geography.to_csv('fake_path', credentials)
os.remove('fake_path')
def test_dataset_download_not_subscribed_but_public(
self, mock_download_stream, mock_get_by_id):
# Given
mock_get_by_id.return_value = test_dataset1 # is public
dataset = Dataset.get(test_dataset1.id)
mock_download_stream.return_value = []
credentials = Credentials('fake_user', '1234')
dataset.to_csv('fake_path', credentials)
os.remove('fake_path')
def test_dataset_subscription_info_default_credentials(self, mocked_credentials, mock_fetch):
# Given
expected_credentials = Credentials('fake_user', '1234')
mocked_credentials.return_value = expected_credentials
dataset = Dataset(db_dataset1)
# When
dataset.subscription_info()
# Then
mock_fetch.assert_called_once_with(db_dataset1['id'], 'dataset', expected_credentials)
def test_do_credentials(self, mocker):
access_token = '1234'
project = 'project'
instant_licensing = True
class DOCredentials:
def __init__(self):
self.access_token = access_token
self.bq_project = project
self.instant_licensing = instant_licensing
mocker.patch('carto.do_token.DoTokenManager.get', return_value=DOCredentials())
credentials = Credentials(self.username, self.api_key)
project_from_do, access_token_from_do = credentials.get_gcloud_credentials()
instant_licensing_from_do = credentials.is_instant_licensing_active()
assert access_token_from_do == access_token
assert project_from_do == project
assert instant_licensing_from_do == instant_licensing
def test_geography_download(self, download_stream_mock, get_by_id_mock,
get_all_mock):
# Given
get_by_id_mock.return_value = test_geography1
geography = Geography.get(test_geography1.id)
get_all_mock.return_value = [geography]
download_stream_mock.return_value = []
credentials = Credentials('fake_user', '1234')
# Then
geography.to_csv('fake_path', credentials)
def test_dataset_download(self, download_stream_mock, get_by_id_mock,
get_all_mock):
# Given
get_by_id_mock.return_value = test_dataset1
dataset = Dataset.get(test_dataset1.id)
get_all_mock.return_value = [dataset]
download_stream_mock.return_value = []
credentials = Credentials('fake_user', '1234')
# Then
dataset.to_csv('fake_path', credentials)
def test_geography_download(self, mocked_bq_client, get_by_id_mock,
get_all_mock):
# Given
get_by_id_mock.return_value = test_geography1
geography = Geography.get(test_geography1.id)
get_all_mock.return_value = [geography]
mocked_bq_client.return_value = BigQueryClientMock()
credentials = Credentials('fake_user', '1234')
# Then
geography.to_csv('fake_path', credentials)
def setUp(self):
if (os.environ.get('APIKEY') is None or os.environ.get('USERNAME') is None):
creds = json.loads(open('tests/e2e/secret.json').read())
self.apikey = creds['APIKEY']
self.username = creds['USERNAME']
else:
self.apikey = os.environ['APIKEY']
self.username = os.environ['USERNAME']
self.credentials = Credentials(self.username, self.apikey)
self.file_path = '/tmp/test_download.csv'
def test_create_api_key_several_sources(self, mocker):
setup_mocks(mocker)
source = Source('fake_table', credentials=Credentials('fakeuser'))
api_key_name = 'fake_name'
auth_api_client = AuthAPIClient()
token, tables = auth_api_client.create_api_key(
[source, source, source], name=api_key_name)
assert token == TOKEN_MOCK
def test_credentials_without_file(self, mocker):
mocker_log = mocker.patch('cartoframes.utils.logger.log.info')
credentials1 = Credentials(self.username, self.api_key)
credentials1.save()
credentials2 = Credentials.from_file()
assert credentials1 == credentials2
mocker_log.assert_called_once_with(
'User credentials for `{0}` were successfully saved to `{1}`'.format(
self.username, DEFAULT_PATH))
credentials1.delete()
with pytest.raises(FileNotFoundError):
Credentials.from_file()
def test_source_get_credentials_base_url(self, mocker):
"""Source should return the correct credentials when base_url is provided"""
setup_mocks(mocker)
source = Source(
'faketable',
credentials=Credentials(base_url='https://fakeuser.carto.com'))
credentials = source.get_credentials()
assert credentials['username'] == 'user'
assert credentials['api_key'] == 'default_public'
assert credentials['base_url'] == 'https://fakeuser.carto.com'
def test_source_get_credentials_username(self, mocker):
"""Source should return the correct credentials when username is provided"""
setup_mocks(mocker)
source = Source('faketable',
credentials=Credentials(username='******',
api_key='1234'))
credentials = source.get_credentials()
assert credentials['username'] == 'fakeuser'
assert credentials['api_key'] == '1234'
assert credentials['base_url'] == 'https://fakeuser.carto.com'
def test_geography_download(self, mock_download_stream, mock_get_by_id,
mock_subscription_ids):
# Given
mock_get_by_id.return_value = test_geography1
geography = Geography.get(test_geography1.id)
mock_download_stream.return_value = []
mock_subscription_ids.return_value = [test_geography1.id]
credentials = Credentials('fake_user', '1234')
# Then
geography.to_csv('fake_path', credentials)
os.remove('fake_path')
