def test_cloud_personal_url():
user1 = APIKeyAuthClient('https://user1.carto.com/a/b/c', API_KEY).username
user2 = APIKeyAuthClient('https://www.user2.carto.com', API_KEY).username
user3 = APIKeyAuthClient('http://www.user3.carto.com/a/b/c',
API_KEY).username
assert user1 == 'user1'
assert user2 == 'user2'
assert user3 == 'user3'
def test_api_key_auth_can_only_read_default_public_with_default_public():
client = APIKeyAuthClient(USER1_BASE_URL, DEFAULT_PUBLIC_API_KEY)
response = client.send('/api/v3/api_keys', 'get')
assert response.status_code == 200
api_keys = response.json()
assert api_keys['count'] == 1
assert len(api_keys['result']) == 1
assert api_keys['result'][0]['name'] == 'Default public'
assert api_keys['result'][0]['token'] == 'default_public'
def test_api_key_auth_can_only_read_default_public_with_default_public():
client = APIKeyAuthClient(USER1_BASE_URL, DEFAULT_PUBLIC_API_KEY)
response = client.send('/api/v3/api_keys', 'get')
assert response.status_code == 200
api_keys = response.json()
assert api_keys['count'] == 1
assert len(api_keys['result']) == 1
assert api_keys['result'][0]['name'] == 'Default public'
assert api_keys['result'][0]['token'] == 'default_public'
def test_on_prem_url():
user1 = APIKeyAuthClient('https://carto.com/user/user1/a/b/c',
API_KEY).username
user2 = APIKeyAuthClient('https://www.carto.com/user/user2',
API_KEY).username
user3 = APIKeyAuthClient('http://www.carto.com/user/user3/a/b/c',
API_KEY).username
assert user1 == 'user1'
assert user2 == 'user2'
assert user3 == 'user3'
def test_user_agent():
expected_user_agent = _ClientIdentifier().get_user_agent()
adapter = requests_mock.Adapter()
session = requests.Session()
# Using file:// cause urllib's urljoin (used in pyrestcli)
# does not support a mock:// schema
session.mount('file', adapter)
adapter.register_uri('POST', 'file://test.carto.com/headers',
request_headers={'User-Agent': expected_user_agent})
client = APIKeyAuthClient('file://test.carto.com', 'some_api_key',
None, session)
client.send('headers', 'post')
def __init__(self, base_url=None, api_key=None, session=None, verbose=0):
self.api_key, self.base_url = _process_credentials(api_key,
base_url)
self.auth_client = APIKeyAuthClient(base_url=self.base_url,
api_key=self.api_key,
session=session)
self.sql_client = SQLClient(self.auth_client)
self.username = self.auth_client.username
self.is_org = self._is_org_user()
self._map_templates = {}
self._srcdoc = None
self._verbose = verbose
def initialize(self):
if not self.api_url and self.user_name:
self.api_url = "https://{}.carto.com/api/".format(self.user_name)
elif not self.api_url and not self.user_name:
raise Exception(
'Not enough data provided to initialize the client')
if self.org_name:
self.client = APIKeyAuthClient(self.api_url, self.api_key,
self.org_name)
else:
self.client = APIKeyAuthClient(self.api_url, self.api_key)
self.sql_client = SQLClient(self.client)
self.batch_client = BatchSQLClient(self.client)
def test_user_agent():
expected_user_agent = _ClientIdentifier().get_user_agent()
adapter = requests_mock.Adapter()
session = requests.Session()
# Using file:// cause urllib's urljoin (used in pyrestcli)
# does not support a mock:// schema
session.mount('file', adapter)
adapter.register_uri('POST',
'file://test.carto.com/headers',
request_headers={'User-Agent': expected_user_agent})
client = APIKeyAuthClient('file://test.carto.com', 'some_api_key', None,
session)
client.send('headers', 'post')
def get_auth_client(username=None,
api_key=None,
baseurl=None,
cdb_client=None):
"""Instantiates a SQL Client from the CARTO Python SDK (v1.0.0)
:param username: CARTO username
:type username: string
:param api_key: API key of CARTO user ``username``
:type api_key: string
:param baseurl: Base URL for CARTO instance (usually suitable for on prem)
:type baseurl: string
:param cdb_client: CARTO Python SDK Authentication client
:type cdb_client: object
:returns: Authenticated SQL client with user credentials
:rtype: sql auth object
"""
from carto.sql import SQLClient
from carto.auth import APIKeyAuthClient
if cdb_client:
sql = SQLClient(cdb_client)
elif username is not None and api_key is not None:
baseurl = get_baseurl(username=username, baseurl=baseurl)
auth_client = APIKeyAuthClient(baseurl, api_key)
sql = SQLClient(auth_client)
else:
raise Exception("`username` and `api_key` or `cdb_client` has to be "
"specified.")
return sql
def setUp(self):
if (os.environ.get('APIKEY') is None
or os.environ.get('USERNAME') is None):
try:
creds = json.loads(open('test/secret.json').read())
self.apikey = creds['APIKEY']
self.username = creds['USERNAME']
except: # noqa
warnings.warn('Skipping CartoContext tests. To test it, '
'create a `secret.json` file in test/ by '
'renaming `secret.json.sample` to `secret.json` '
'and updating the credentials to match your '
'environment.')
self.apikey = None
self.username = None
else:
self.apikey = os.environ['APIKEY']
self.username = os.environ['USERNAME']
if self.username and self.apikey:
self.baseurl = 'https://{username}.carto.com/'.format(
username=self.username)
self.auth_client = APIKeyAuthClient(base_url=self.baseurl,
api_key=self.apikey)
self.sql_client = SQLClient(self.auth_client)
# sets skip value
WILL_SKIP = self.apikey is None or self.username is None # noqa: F841
has_mpl = 'mpl' if os.environ.get('MPLBACKEND') else 'nonmpl'
pyver = sys.version[0:3].replace('.', '_')
# for writing to carto
self.test_write_lnglat_table = (
'cartoframes_test_write_lnglat_table_{ver}_{mpl}'.format(
ver=pyver, mpl=has_mpl))
def user():
"""
Handy way for tests to have access to a user object
:return: User instance that correspond to the test user
"""
return UserManager(APIKeyAuthClient(BASE_URL, API_KEY, ORGANIZATION)). \
get(USERNAME) if ORGANIZATION is not None else None
def _create_auth_client(credentials, public=False):
return APIKeyAuthClient(
base_url=credentials.base_url,
api_key='default_public' if public else credentials.api_key,
session=credentials.session,
client_id='cartoframes_{}'.format(__version__),
user_agent='cartoframes_{}'.format(__version__))
def setUp(self):
if (os.environ.get('APIKEY') is None
or os.environ.get('USERNAME') is None):
try:
creds = json.loads(open('test/secret.json').read())
self.apikey = creds['APIKEY']
self.username = creds['USERNAME']
except: # noqa: E722
warnings.warn("Skipping CartoContext tests. To test it, "
"create a `secret.json` file in test/ by "
"renaming `secret.json.sample` to `secret.json` "
"and updating the credentials to match your "
"environment.")
self.apikey = None
self.username = None
else:
self.apikey = os.environ['APIKEY']
self.username = os.environ['USERNAME']
self.user_url = self.user_url()
if self.username and self.apikey:
self.baseurl = self.user_url.format(username=self.username)
self.auth_client = APIKeyAuthClient(base_url=self.baseurl,
api_key=self.apikey)
self.sql_client = SQLClient(self.auth_client)
# sets skip value
WILL_SKIP = self.apikey is None or self.username is None # noqa: F841
self.points = 'tweets_obama'
self.polys = 'nat'
self.local = 'cb_2013_us_csa_500k'
def update_asset_on_carto(asset_dict, fields):
auth_client = APIKeyAuthClient(api_key=CARTO_API_KEY,
base_url=USR_BASE_URL)
sql = SQLClient(auth_client)
#values_tuple_strings = [make_values_tuple_string_from_model(r, fields) for r in [asset]]
# OR POSSIBLY
#values_tuple_strings = [make_values_tuple_string_from_model(asset, fields)]
#q = f"UPDATE {TABLE_NAME} SET {values_tuple_strings} WHERE asset_id = {asset.id};"
#values_tuple_strings = [values_string_from_model(asset, fields)]
#q = f"UPDATE {TABLE_NAME} SET ({', '.join(fields + ['the_geom', 'the_geom_webmercator'])}) " \
# f"VALUES {', '.join(map(lambda x: x + 1, values_tuple_strings))};" # This is throwing an
# error, and it's really not clear why it's trying to map a math function over strings.
# Let's ignore the the_geom* fields for now and do the update the simple way:
# Single updates can be done like this:
# UPDATE election_results SET votes=52, pro=24 WHERE county_id = 1;
other_fields = copy.deepcopy(fields)
other_fields.remove('id')
q = f"UPDATE {TABLE_NAME} SET {set_string_from_model(asset_dict, other_fields)} WHERE id = {asset_dict['asset'].id};"
assert len(q) < 16384
print(q)
results = sql.send(q)
def __init__(self, user, api_key, options={}):
super().__init__(options)
self.do_post = options.get('do_post', False)
self.parse_json = options.get('parse_json', True)
self.format = options.get('format', 'json')
self.base_url_option = options.get('base_url', '')
self.api_version = options.get('api_version', self.DEFAULT_API_VERSION)
self.batch = options.get('batch', False)
self.user = user
self.api_key = api_key
self.base_url = self._generate_base_url(user, self.base_url_option)
# Carto Context for DataFrame handling
self._carto_context = None
# Carto client for COPYs
self._copy_client = None
self._auth_client = APIKeyAuthClient(api_key=api_key,
base_url=self.base_url)
self._sql_client = SQLClient(self._auth_client,
api_version=self.api_version)
self._batch_client = None
if self.batch:
self._batch_client = BatchSQLClient(self._auth_client)
def fix_carto_geofields(asset_id=None):
auth_client = APIKeyAuthClient(api_key=CARTO_API_KEY,
base_url=USR_BASE_URL)
sql = SQLClient(auth_client)
# Now the problem with pushing this data through SQL calls is that Carto does not rerun the
# processes that add values for the_geom and the_geom_webmercator. So it kind of seems like
# we have to do this ourselves as documented at
# https://gis.stackexchange.com/a/201908
q = f"UPDATE {TABLE_NAME} SET the_geom = ST_SetSRID(st_makepoint(longitude, latitude),4326)"
if asset_id is not None:
q += f" WHERE id = {asset_id}" # This can significantly speed up Carto geofield updates
# when saving a single model instance.
# This works because 'longitude' and 'latitude' are the names of the corresponding fields in the CSV file.
results1 = sql.send(q) # This takes 12 seconds to run for 100,000 rows.
# Exporting the data immediately after this is run oddly leads to the same CSV file as exporting before
# it is run, but waiting a minute and exporting again gives something with the_geom values in the same
# rows as the table on the Carto site. Basically, the exported CSV file can lag the view on the Carto
# web site by a minute or two.
q = f"SELECT ST_Transform(ST_SetSRID(st_makepoint(longitude, latitude),4326),3857) as the_geom_webmercator FROM {TABLE_NAME}"
results2 = sql.send(q) # This one ran much faster.
# One improvement is that you can replace ST_SetSRID(st_makepoint(lon, lat)) with CDB_LatLng(lat, lon)
# though I don't know if it leads to any performance improvement.
print(
f"Tried to add values for the the_geom and the_geom_webmercator fields in {TABLE_NAME}. The requests completed in {results1['time']} s and {results2['time']} s."
)
def api_key_auth_client_usr():
"""
Returns a API key authentication client that can be used to send
authenticated test requests to CARTO
:return: APIKeyAuthClient instance
"""
return APIKeyAuthClient(USR_BASE_URL, API_KEY)
def setUp(self):
if (os.environ.get('APIKEY') is None
or os.environ.get('USERNAME') is None):
try:
creds = json.loads(open('test/secret.json').read())
self.apikey = creds['APIKEY']
self.username = creds['USERNAME']
except:
warnings.warn("Skipping CartoContext tests. To test it, "
"create a `secret.json` file in test/ by "
"renaming `secret.json.sample` to `secret.json` "
"and updating the credentials to match your "
"environment.")
self.apikey = None
self.username = None
else:
self.apikey = os.environ['APIKEY']
self.username = os.environ['USERNAME']
if self.username and self.apikey:
self.baseurl = 'https://{username}.carto.com/'.format(
username=self.username)
self.auth_client = APIKeyAuthClient(base_url=self.baseurl,
api_key=self.apikey)
self.sql_client = SQLClient(self.auth_client)
# sets skip value
WILL_SKIP = self.apikey is None or self.username is None
# table naming info
has_mpl = 'mpl' if os.environ.get('MPLBACKEND') else 'nonmpl'
pyver = sys.version[0:3].replace('.', '_')
# test tables
self.test_read_table = 'cb_2013_us_csa_500k'
self.valid_columns = set([
'affgeoid', 'aland', 'awater', 'created_at', 'csafp', 'geoid',
'lsad', 'name', 'the_geom', 'updated_at'
])
# for writing to carto
self.test_write_table = 'cartoframes_test_table_{ver}_{mpl}'.format(
ver=pyver, mpl=has_mpl)
# for batch writing to carto
self.test_write_batch_table = (
'cartoframes_test_batch_table_{ver}_{mpl}'.format(ver=pyver,
mpl=has_mpl))
self.test_write_lnglat_table = (
'cartoframes_test_write_lnglat_table_{ver}_{mpl}'.format(
ver=pyver, mpl=has_mpl))
# for queries
self.test_query_table = ('cartoframes_test_query_'
'table_{ver}_{mpl}'.format(ver=pyver,
mpl=has_mpl))
self.test_delete_table = ('cartoframes_test_delete_'
'table_{ver}_{mpl}').format(ver=pyver,
mpl=has_mpl)
def get_carto_client(api_key, base_url):
"""
Returns APIKeyAuthClient object for CartoDB Python SDK.
Parameters:
- api_key: string
- base_url: string
"""
return APIKeyAuthClient(api_key=api_key, base_url=base_url)
def wrong_onprem_auth_client():
"""
Returns an authentication client that can be used to send authenticated
test requests to CARTO onpremises
:return: NonVerifiedAPIKeyAuthClient instance
"""
if ONPREM_API_KEY is None:
return None
return APIKeyAuthClient(ONPREMISES_USR_BASE_URL, ONPREM_API_KEY,
ONPREM_ORGANIZATION)
def get_api_key_auth_client(self):
if not self._api_key_auth_client:
self._api_key_auth_client = APIKeyAuthClient(
base_url=self._base_url,
api_key=self.api_key,
session=self.session,
client_id='cartoframes_{}'.format(__version__),
user_agent='cartoframes_{}'.format(__version__))
return self._api_key_auth_client
def get_carto_asset_ids(id_to_check=None):
auth_client = APIKeyAuthClient(api_key=CARTO_API_KEY,
base_url=USR_BASE_URL)
sql = SQLClient(auth_client)
if id_to_check is None:
results = sql.send(f"SELECT id FROM {TABLE_NAME}")
else:
results = sql.send(
f"SELECT id FROM {TABLE_NAME} WHERE id = {id_to_check}")
ids = [r['id'] for r in results['rows']]
return ids
def delete_map(map_name):
auth_client = APIKeyAuthClient(CARTO_BASE_URL, CARTO_API_KEY)
named_map_manager = NamedMapManager(auth_client)
try:
named_map = named_map_manager.get(map_name)
if named_map is not None:
named_map.client = auth_client
named_map.delete()
except Exception as e:
#ignore
print(e)
def create_map(iso, lon, lat):
creds = Credentials(username=CARTO_USER_NAME, api_key=CARTO_API_KEY)
set_default_credentials(creds)
dataset_name = 'onekmiso'
dataset_name = dataset_name + str(int(round(time.time() * 1000)))
to_carto(iso, dataset_name, if_exists='replace', log_enabled=True)
auth_client = APIKeyAuthClient(CARTO_BASE_URL, CARTO_API_KEY)
FACTOR = DPI / 72.0
map_name = 'tpl_' + dataset_name
create_named_map(auth_client, CARTO_USER_NAME, dataset_name, map_name, FACTOR, lon, lat)
return map_name
def __init__(self):
self.carto_api_key = os.environ['API_KEY']
self.carto_account = os.environ['ACCOUNT']
USR_BASE_URL = "https://{user}.carto.com/".format(
user=self.carto_account)
self.auth_client = APIKeyAuthClient(api_key=self.carto_api_key,
base_url=USR_BASE_URL)
#this mimics the carto docs, leave it this way
self.sql = SQLClient(self.auth_client)
self.dataset_manager = DatasetManager(self.auth_client)
def __init__(self, CARTO_USER, CARTO_API_URL, CARTO_ORG, CARTO_API_KEY,
USER_QUOTA):
self.CARTO_USER = CARTO_USER
self.CARTO_ORG = CARTO_ORG
self.USER_QUOTA = USER_QUOTA
### CARTO clients
auth_client = APIKeyAuthClient(CARTO_API_URL, CARTO_API_KEY, CARTO_ORG)
self.sql = SQLClient(auth_client)
self.vm = VisualizationManager(auth_client)
self.dm = DatasetManager(auth_client)
### logger, variables and CARTO clients
self.logger = logging.getLogger('carto_report')
self.logger.addHandler(logging.NullHandler())
def upload_to_carto(csv_loc):
print('Uploading processed data to Carto.')
#set up carto authentication using local variables for username (CARTO_WRI_RW_USER) and API key (CARTO_WRI_RW_KEY)
auth_client = APIKeyAuthClient(api_key=os.getenv('CARTO_WRI_RW_KEY'),
base_url="https://{user}.carto.com/".format(
user=os.getenv('CARTO_WRI_RW_USER')))
#set up dataset manager with authentication
dataset_manager = DatasetManager(auth_client)
#upload dataset to carto
dataset = dataset_manager.create(csv_loc)
print('Carto table created: {}'.format(
os.path.basename(csv_loc).split('.')[0]))
#set dataset privacy to 'Public with link'
dataset.privacy = 'LINK'
dataset.save()
print('Privacy set to public with link.')
def index():
form = MainForm()
if form.validate_on_submit():
auth_client = APIKeyAuthClient(form.carto_base_url.data, form.carto_api_key.data)
if form.incident_type.data == "alerts":
incidents = AlertProcessor(auth_client, username=form.db_username.data, password=form.db_password.data, host=form.db_host.data)
elif form.incident_type.data == "jams":
incidents = JamProcessor(auth_client, username=form.db_username.data, password=form.db_password.data, host=form.db_host.data)
else:
flash("Unknown incident type: {incident_type}".format(incident_type=form.incident_type.data), "error")
return render_template("index.html", form=form)
with io.StringIO() as csv_out:
try:
incidents.get_values(csv_out, pub_utc_date__gt=form.from_timestamp.data, pub_utc_date__lt=form.to_timestamp.data)
except psycopg2.OperationalError as e:
form.db_host.errors = True
flash(e, "error")
return render_template("index.html", form=form)
else:
csv_out.seek(0)
with io.BytesIO(bytes(csv_out.read(), "utf-8")) as csv_in:
table_name = form.carto_table_name.data or get_table_name()
try:
incidents.create_table(table_name=table_name, cartodbfy=True)
except CartoException as e:
form.carto_base_url.errors = True
flash(e, "error")
return render_template("index.html", form=form)
try:
incidents.append_data(csv_in, table_name=table_name)
except CartoException as e:
flash(e, "error")
return render_template("index.html", form=form)
flash("Data has been saved in CARTO ({table_name})".format(table_name=table_name), "success")
else:
if form.errors:
flash(form.errors, "error")
return render_template("index.html", form=form)
def conn(self):
if self._conn is None:
self.logger.info('Making connection to Carto {} account...'.format(
self.user))
try:
api_key = self.api_key
base_url = USR_BASE_URL.format(user=self.user)
auth_client = APIKeyAuthClient(api_key=api_key,
base_url=base_url)
conn = SQLClient(auth_client)
self._conn = conn
self.logger.info('Connected to Carto.\n')
except CartoException as e:
self.logger.error(
'Failed making connection to Carto {} account...'.format(
self.user))
raise e
return self._conn
def upload_to_carto(file, privacy):
'''
Upload tables to Carto
INPUT file: location of file on local computer that you want to upload (string)
'''
# set up carto authentication using local variables for username (CARTO_WRI_RW_USER) and API key (CARTO_WRI_RW_KEY)
auth_client = APIKeyAuthClient(api_key=os.getenv('CARTO_WRI_RW_KEY'),
base_url="https://{user}.carto.com/".format(
user=os.getenv('CARTO_WRI_RW_USER')))
# set up dataset manager with authentication
dataset_manager = DatasetManager(auth_client)
# upload dataset to carto
dataset = dataset_manager.create(file)
logger.info('Carto table created: {}'.format(
os.path.basename(file).split('.')[0]))
# set dataset privacy to 'Public with link'
dataset.privacy = privacy
logger.info('Privacy set to public with link.')
dataset.save()
def __init__(self, config_path):
"""
Initializes the instance
Args:
config_path: the path to the configuration file
"""
self.logger = logging.getLogger('sb')
"""The instance logger"""
self.config = self.load_config(config_path)
"""Configuration dict"""
self.logger.setLevel(self.config['log_level'])
self.logger.debug('Configuration processed')
creds = self.config["carto_account"]
self.logger.debug(creds)
self.auth_client = APIKeyAuthClient(creds["api_url"], creds["api_key"],
creds["organization"])
"""API key CARTO authentication client"""
def dumpToCarto(eleList, table_name):
# I am using my CARTO account
USERNAME = "******"
USR_BASE_URL = "https://{user}.carto.com/".format(user=USERNAME)
auth_client = APIKeyAuthClient(
api_key="53bb19efc968a08f7bdc2c1ffc29c31659240b39",
base_url=USR_BASE_URL)
sql = SQLClient(auth_client)
table_name = 'strava_segments_' + table_name
for segment in eleList:
try:
query = "UPDATE {table} SET cartodb_id={id}, the_geom=ST_SetSRID(ST_MakePoint({long}, {lat}),4326), name='{name}', value={value}, date=now() WHERE cartodb_id={id}". \
format(table=table_name,id=segment[0],long=segment[7],lat=segment[8],name=segment[1],value=segment[2])
logger.info(query)
sql.send(query)
except CartoException as e:
logger.error(e)
def test_client_id_in_requests():
expected_client_id = _ClientIdentifier().get_client_identifier()
client = APIKeyAuthClient('https://test.carto.com', 'some_api_key')
http_method, requests_args = client.prepare_send('post')
assert requests_args['params']['client'] == expected_client_id
def test_api_key_auth_client_me_endpoint():
client = APIKeyAuthClient(USER1_BASE_URL, API_KEY)
username = client.send('/api/v3/me', 'get').json()['config']['user_name']
assert username == USER1_USERNAME
