def test_dict_items(self):
"""utils.dict_items"""
# ensure correct formation of dict items from provided styling
polygon_style_dict = dict_items(self.polygon_style)
self.assertDictEqual(OrderedDict(polygon_style_dict),
self.polygon_style,
msg="pollygon styling")
# point style
point_style_dict = dict_items(self.point_style)
self.assertDictEqual(OrderedDict(point_style_dict),
self.point_style,
msg="point styling")
# multi layer styling
complex_style_dict = dict_items(self.complex_style)
self.assertDictEqual(OrderedDict(complex_style_dict),
self.complex_style,
msg="multi-layer styling")
def _debug_print(self, **kwargs):
if self._verbose <= 0:
return
for key, value in dict_items(kwargs):
if isinstance(value, requests.Response):
str_value = "status_code: {status_code}, content: {content}".format(
status_code=value.status_code, content=value.content)
else:
str_value = str(value)
if self._verbose < 2 and len(str_value) > 300:
str_value = '{}\n\n...\n\n{}'.format(str_value[:250], str_value[-50:])
print('{key}: {value}'.format(key=key,
value=str_value))
def map(self, layers=None, interactive=True,
zoom=None, lat=None, lng=None, size=(800, 400),
ax=None):
"""Produce a CARTO map visualizing data layers.
Example:
Create a map with two data layers, and one BaseMap layer.
::
import cartoframes
from cartoframes import Layer, BaseMap, styling
cc = cartoframes.CartoContext(BASEURL, APIKEY)
cc.map(layers=[BaseMap(),
Layer('acadia_biodiversity',
color={'column': 'simpson_index',
'scheme': styling.tealRose(7)}),
Layer('peregrine_falcon_nest_sites',
size='num_eggs',
color={'column': 'bird_id',
'scheme': styling.vivid(10))],
interactive=True)
Args:
layers (list, optional): List of one or more of the following:
- Layer: cartoframes Layer object for visualizing data from a
CARTO table. See `layer.Layer <#layer.Layer>`__ for all
styling options.
- BaseMap: Basemap for contextualizng data layers. See
`layer.BaseMap <#layer.BaseMap>`__ for all styling options.
- QueryLayer: Layer from an arbitrary query. See
`layer.QueryLayer <#layer.QueryLayer>`__ for all styling
options.
interactive (bool, optional): Defaults to ``True`` to show an
interactive slippy map. Setting to ``False`` creates a static
map.
zoom (int, optional): Zoom level of map. Acceptable values are
usually in the range 0 to 19. 0 has the entire earth on a
single tile (256px square). Zoom 19 is the size of a city
block. Must be used in conjunction with ``lng`` and ``lat``.
Defaults to a view to have all data layers in view.
lat (float, optional): Latitude value for the center of the map.
Must be used in conjunction with ``zoom`` and ``lng``. Defaults
to a view to have all data layers in view.
lng (float, optional): Longitude value for the center of the map.
Must be used in conjunction with ``zoom`` and ``lat``. Defaults
to a view to have all data layers in view.
size (tuple, optional): List of pixel dimensions for the map. Format
is ``(width, height)``. Defaults to ``(800, 400)``.
Returns:
IPython.display.HTML: Interactive maps are rendered in an ``iframe``,
while static maps are rendered in ``img`` tags.
"""
# TODO: add layers preprocessing method like
# layers = process_layers(layers)
# that uses up to layer limit value error
if not hasattr(IPython, 'display'):
raise NotImplementedError('Nope, cannot display maps at the '
'command line.')
if layers is None:
layers = []
elif not isinstance(layers, collections.Iterable):
layers = [layers]
else:
layers = list(layers)
if len(layers) > 8:
raise ValueError('map can have at most 8 layers')
if any([zoom, lat, lng]) != all([zoom, lat, lng]):
raise ValueError('zoom, lat, and lng must all or none be provided')
# When no layers are passed, set default zoom
if ((len(layers) == 0 and zoom is None) or
(len(layers) == 1 and layers[0].is_basemap)):
[zoom, lat, lng] = [3, 38, -99]
has_zoom = zoom is not None
# Check basemaps, add one if none exist
base_layers = [idx for idx, layer in enumerate(layers)
if layer.is_basemap]
if len(base_layers) > 1:
raise ValueError('map can at most take 1 BaseMap layer')
if len(base_layers) > 0:
layers.insert(0, layers.pop(base_layers[0]))
else:
layers.insert(0, BaseMap())
# Check for a time layer, if it exists move it to the front
time_layers = [idx for idx, layer in enumerate(layers)
if not layer.is_basemap and layer.time]
time_layer = layers[time_layers[0]] if len(time_layers) > 0 else None
if len(time_layers) > 1:
raise ValueError('Map can at most take 1 Layer with time '
'column/field')
if time_layer:
raise NotImplementedError('Animated maps are not yet supported')
if not interactive:
raise ValueError('map cannot display a static image with a '
'time_column')
layers.append(layers.pop(time_layers[0]))
# If basemap labels are on front, add labels layer
basemap = layers[0]
if basemap.is_basic() and basemap.labels == 'front':
layers.append(BaseMap(basemap.source,
labels=basemap.labels,
only_labels=True))
# Setup layers
for idx, layer in enumerate(layers):
layer._setup(layers, idx)
nb_layers = non_basemap_layers(layers)
options = {'basemap_url': basemap.url}
for idx, layer in enumerate(nb_layers):
self._check_query(layer.query,
style_cols=layer.style_cols)
options['cartocss_' + str(idx)] = layer.cartocss
options['sql_' + str(idx)] = layer.query
params = {
'config': json.dumps(options),
'anti_cache': random.random(),
}
if has_zoom:
params.update({'zoom': zoom, 'lat': lat, 'lon': lng})
options.update({'zoom': zoom, 'lat': lat, 'lng': lng})
else:
options.update(self._get_bounds(nb_layers))
map_name = self._send_map_template(layers, has_zoom=has_zoom)
api_url = '{base_url}api/v1/map'.format(base_url=self.base_url)
static_url = ('{api_url}/static/named/{map_name}'
'/{width}/{height}.png?{params}').format(
api_url=api_url,
map_name=map_name,
width=size[0],
height=size[1],
params=urlencode(params))
html = '<img src="{url}" />'.format(url=static_url)
# TODO: write this as a private method
if interactive:
netloc = urlparse(self.base_url).netloc
domain = 'carto.com' if netloc.endswith('.carto.com') else netloc
def safe_quotes(text, escape_single_quotes=False):
"""htmlify string"""
if isinstance(text, str):
safe_text = text.replace('"', """)
if escape_single_quotes:
safe_text = safe_text.replace("'", "\'")
return safe_text.replace('True', 'true')
return text
config = {
'user_name': self.username,
'maps_api_template': self.base_url[:-1],
'sql_api_template': self.base_url[:-1],
'tiler_protocol': 'https',
'tiler_domain': domain,
'tiler_port': '80',
'type': 'torque' if time_layer else 'namedmap',
'named_map': {
'name': map_name,
'params': {
k: safe_quotes(v, escape_single_quotes=True)
for k, v in dict_items(options)
},
},
}
map_options = {
'filter': ['http', 'mapnik', 'torque'],
'https': True,
}
if time_layer:
config.update({
'order': 1,
'options': {
'query': time_layer.query,
'user_name': self.username,
'tile_style': time_layer.torque_cartocss,
}
})
config['named_map'].update({
'layers': [{
'layer_name': 't',
}],
})
map_options.update({
'time_slider': True,
'loop': True,
})
bounds = [] if has_zoom else [[options['north'], options['east']],
[options['south'], options['west']]]
content = self._get_iframe_srcdoc(config=config,
bounds=bounds,
options=options,
map_options=map_options)
img_html = html
html = (
'<iframe srcdoc="{content}" width={width} height={height}>'
' Preview image: {img_html}'
'</iframe>'
).format(content=safe_quotes(content),
width=size[0],
height=size[1],
img_html=img_html)
return IPython.display.HTML(html)
else:
try:
import matplotlib.image as mpi
import matplotlib.pyplot as plt
except ImportError:
warn('Matplotlib not detected. Saving image directly to disk')
raise NotImplementedError
raw_data = mpi.imread(static_url)
f = plt.gcf()
if ax is None:
ax = plt.gca()
ax.imshow(raw_data)
ax.axis('off')
return ax
def test_cartocontext_write(self):
"""context.CartoContext.write normal usage"""
from cartoframes.context import MAX_ROWS_LNGLAT
cc = cartoframes.CartoContext(base_url=self.baseurl,
api_key=self.apikey)
data = {
'nums': list(range(100, 0, -1)),
'category':
[random.choice('abcdefghijklmnop') for _ in range(100)],
'lat': [0.01 * i for i in range(100)],
'long': [-0.01 * i for i in range(100)]
}
schema = {
'nums': int,
'category': 'object',
'lat': float,
'long': float
}
df = pd.DataFrame(data).astype(schema)
cc.write(df, self.test_write_table)
# check if table exists
resp = self.sql_client.send('''
SELECT *
FROM {table}
LIMIT 0
'''.format(table=self.test_write_table))
self.assertIsNotNone(resp)
# check that table has same number of rows
resp = self.sql_client.send('''
SELECT count(*)
FROM {table}'''.format(table=self.test_write_table))
self.assertEqual(resp['rows'][0]['count'], len(df))
# should error for existing table
with self.assertRaises(NameError):
cc.write(df, self.test_read_table, overwrite=False)
# overwrite table and create the_geom column
cc.write(df,
self.test_write_table,
overwrite=True,
lnglat=('long', 'lat'))
resp = self.sql_client.send('''
SELECT count(*) AS num_rows, count(the_geom) AS num_geoms
FROM {table}
'''.format(table=self.test_write_table))
# number of geoms should equal number of rows
self.assertEqual(resp['rows'][0]['num_rows'],
resp['rows'][0]['num_geoms'])
# test batch lnglat behavior
n_rows = MAX_ROWS_LNGLAT + 1
df = pd.DataFrame({
'latvals': [random.random() for r in range(n_rows)],
'lngvals': [random.random() for r in range(n_rows)]
})
job = cc.write(df,
self.test_write_lnglat_table,
lnglat=('lngvals', 'latvals'))
self.assertIsInstance(job, cartoframes.context.BatchJobStatus)
# test batch writes
n_rows = 550000
df = pd.DataFrame({'vals': [random.random() for r in range(n_rows)]})
cc.write(df, self.test_write_batch_table)
resp = self.sql_client.send('''
SELECT count(*) AS num_rows FROM {table}
'''.format(table=self.test_write_batch_table))
# number of rows same in dataframe and carto table
self.assertEqual(n_rows, resp['rows'][0]['num_rows'])
cols = self.sql_client.send('''
SELECT * FROM {table} LIMIT 1
'''.format(table=self.test_write_batch_table))
expected_schema = {
'vals': {
'type': 'number'
},
'the_geom': {
'type': 'geometry'
},
'the_geom_webmercator': {
'type': 'geometry'
},
'cartodb_id': {
'type': 'number'
}
}
# table should be properly created
# util columns + new column of type number
self.assertDictEqual(cols['fields'], expected_schema)
# test properly encoding
df = pd.DataFrame({'vals': [1, 2], 'strings': ['a', 'ô']})
cc.write(df, self.test_write_table, overwrite=True)
# check if table exists
resp = self.sql_client.send('''
SELECT *
FROM {table}
LIMIT 0
'''.format(table=self.test_write_table))
self.assertIsNotNone(resp)
cc.delete(self.test_write_table)
df = pd.DataFrame({'vals': list('abcd'), 'ids': list('wxyz')})
df = df.astype({'vals': str, 'ids': str})
cc.write(df, self.test_write_table)
schema = cc.sql_client.send('select ids, vals from {}'.format(
self.test_write_table))['fields']
self.assertSetEqual(set([schema[c]['type'] for c in schema]),
set(('string', )))
df = pd.DataFrame({
'vals': list('abcd'),
'ids': list('wxyz'),
'nums': [1.2 * i for i in range(4)],
'boolvals': [
True,
False,
None,
True,
],
})
cc.write(df,
self.test_write_table,
overwrite=True,
type_guessing='true')
resp = cc.sql_client.send('SELECT * FROM {}'.format(
self.test_write_table))['fields']
schema = {k: v['type'] for k, v in dict_items(resp)}
ans = dict(vals='string',
ids='string',
nums='number',
boolvals='boolean',
the_geom='geometry',
the_geom_webmercator='geometry',
cartodb_id='number')
self.assertDictEqual(schema, ans)
def test_cartocontext_write(self):
"""context.CartoContext.write normal usage"""
cc = cartoframes.CartoContext(base_url=self.baseurl,
api_key=self.apikey)
data = {'nums': list(range(100, 0, -1)),
'category': [random.choice('abcdefghijklmnop')
for _ in range(100)],
'lat': [0.01 * i for i in range(100)],
'long': [-0.01 * i for i in range(100)]}
schema = {'nums': int,
'category': 'object',
'lat': float,
'long': float}
df = pd.DataFrame(data).astype(schema)
dataset = cc.write(df, self.test_write_table)
self.test_write_table = dataset.table_name
# check if table exists
resp = self.sql_client.send('''
SELECT *
FROM {table}
LIMIT 0
'''.format(table=self.test_write_table))
self.assertIsNotNone(resp)
# check that table has same number of rows
resp = self.sql_client.send('''
SELECT count(*)
FROM {table}'''.format(table=self.test_write_table))
self.assertEqual(resp['rows'][0]['count'], len(df))
# should error for existing table
err_msg = ('Table with name {t} and schema {s} already exists in CARTO. Please choose a different `table_name`'
'or use if_exists="replace" to overwrite it').format(t=self.test_write_table, s='public')
with self.assertRaises(CartoException, msg=err_msg):
cc.write(df, self.test_read_table, overwrite=False)
# overwrite table and create the_geom column
cc.write(df, self.test_write_table,
overwrite=True,
lnglat=('long', 'lat'))
resp = self.sql_client.send('''
SELECT count(*) AS num_rows, count(the_geom) AS num_geoms
FROM {table}
'''.format(table=self.test_write_table))
# number of geoms should equal number of rows
self.assertEqual(resp['rows'][0]['num_rows'],
resp['rows'][0]['num_geoms'])
cc.delete(self.test_write_table)
df = pd.DataFrame({'vals': list('abcd'), 'ids': list('wxyz')})
df = df.astype({'vals': str, 'ids': str})
cc.write(df, self.test_write_table, overwrite=True)
schema = cc.sql_client.send('select ids, vals from {}'.format(
self.test_write_table))['fields']
self.assertSetEqual(set([schema[c]['type'] for c in schema]),
set(('string', )))
df = pd.DataFrame({'vals': list('abcd'),
'ids': list('wxyz'),
'nums': [1.2 * i for i in range(4)],
'boolvals': [True, False, None, True, ],
})
df['boolvals'] = df['boolvals'].astype(bool)
cc.write(df, self.test_write_table, overwrite=True)
resp = cc.sql_client.send('SELECT * FROM {}'.format(
self.test_write_table))['fields']
schema = {k: v['type'] for k, v in dict_items(resp)}
ans = dict(vals='string', ids='string', nums='number',
boolvals='boolean', the_geom='geometry',
the_geom_webmercator='geometry', cartodb_id='number')
self.assertDictEqual(schema, ans)
