def test_dict_items(self): """utils.dict_items""" # ensure correct formation of dict items from provided styling polygon_style_dict = dict_items(self.polygon_style) self.assertDictEqual(OrderedDict(polygon_style_dict), self.polygon_style, msg="pollygon styling") # point style point_style_dict = dict_items(self.point_style) self.assertDictEqual(OrderedDict(point_style_dict), self.point_style, msg="point styling") # multi layer styling complex_style_dict = dict_items(self.complex_style) self.assertDictEqual(OrderedDict(complex_style_dict), self.complex_style, msg="multi-layer styling")
def _debug_print(self, **kwargs): if self._verbose <= 0: return for key, value in dict_items(kwargs): if isinstance(value, requests.Response): str_value = "status_code: {status_code}, content: {content}".format( status_code=value.status_code, content=value.content) else: str_value = str(value) if self._verbose < 2 and len(str_value) > 300: str_value = '{}\n\n...\n\n{}'.format(str_value[:250], str_value[-50:]) print('{key}: {value}'.format(key=key, value=str_value))
def map(self, layers=None, interactive=True, zoom=None, lat=None, lng=None, size=(800, 400), ax=None): """Produce a CARTO map visualizing data layers. Example: Create a map with two data layers, and one BaseMap layer. :: import cartoframes from cartoframes import Layer, BaseMap, styling cc = cartoframes.CartoContext(BASEURL, APIKEY) cc.map(layers=[BaseMap(), Layer('acadia_biodiversity', color={'column': 'simpson_index', 'scheme': styling.tealRose(7)}), Layer('peregrine_falcon_nest_sites', size='num_eggs', color={'column': 'bird_id', 'scheme': styling.vivid(10))], interactive=True) Args: layers (list, optional): List of one or more of the following: - Layer: cartoframes Layer object for visualizing data from a CARTO table. See `layer.Layer <#layer.Layer>`__ for all styling options. - BaseMap: Basemap for contextualizng data layers. See `layer.BaseMap <#layer.BaseMap>`__ for all styling options. - QueryLayer: Layer from an arbitrary query. See `layer.QueryLayer <#layer.QueryLayer>`__ for all styling options. interactive (bool, optional): Defaults to ``True`` to show an interactive slippy map. Setting to ``False`` creates a static map. zoom (int, optional): Zoom level of map. Acceptable values are usually in the range 0 to 19. 0 has the entire earth on a single tile (256px square). Zoom 19 is the size of a city block. Must be used in conjunction with ``lng`` and ``lat``. Defaults to a view to have all data layers in view. lat (float, optional): Latitude value for the center of the map. Must be used in conjunction with ``zoom`` and ``lng``. Defaults to a view to have all data layers in view. lng (float, optional): Longitude value for the center of the map. Must be used in conjunction with ``zoom`` and ``lat``. Defaults to a view to have all data layers in view. size (tuple, optional): List of pixel dimensions for the map. Format is ``(width, height)``. Defaults to ``(800, 400)``. Returns: IPython.display.HTML: Interactive maps are rendered in an ``iframe``, while static maps are rendered in ``img`` tags. """ # TODO: add layers preprocessing method like # layers = process_layers(layers) # that uses up to layer limit value error if not hasattr(IPython, 'display'): raise NotImplementedError('Nope, cannot display maps at the ' 'command line.') if layers is None: layers = [] elif not isinstance(layers, collections.Iterable): layers = [layers] else: layers = list(layers) if len(layers) > 8: raise ValueError('map can have at most 8 layers') if any([zoom, lat, lng]) != all([zoom, lat, lng]): raise ValueError('zoom, lat, and lng must all or none be provided') # When no layers are passed, set default zoom if ((len(layers) == 0 and zoom is None) or (len(layers) == 1 and layers[0].is_basemap)): [zoom, lat, lng] = [3, 38, -99] has_zoom = zoom is not None # Check basemaps, add one if none exist base_layers = [idx for idx, layer in enumerate(layers) if layer.is_basemap] if len(base_layers) > 1: raise ValueError('map can at most take 1 BaseMap layer') if len(base_layers) > 0: layers.insert(0, layers.pop(base_layers[0])) else: layers.insert(0, BaseMap()) # Check for a time layer, if it exists move it to the front time_layers = [idx for idx, layer in enumerate(layers) if not layer.is_basemap and layer.time] time_layer = layers[time_layers[0]] if len(time_layers) > 0 else None if len(time_layers) > 1: raise ValueError('Map can at most take 1 Layer with time ' 'column/field') if time_layer: raise NotImplementedError('Animated maps are not yet supported') if not interactive: raise ValueError('map cannot display a static image with a ' 'time_column') layers.append(layers.pop(time_layers[0])) # If basemap labels are on front, add labels layer basemap = layers[0] if basemap.is_basic() and basemap.labels == 'front': layers.append(BaseMap(basemap.source, labels=basemap.labels, only_labels=True)) # Setup layers for idx, layer in enumerate(layers): layer._setup(layers, idx) nb_layers = non_basemap_layers(layers) options = {'basemap_url': basemap.url} for idx, layer in enumerate(nb_layers): self._check_query(layer.query, style_cols=layer.style_cols) options['cartocss_' + str(idx)] = layer.cartocss options['sql_' + str(idx)] = layer.query params = { 'config': json.dumps(options), 'anti_cache': random.random(), } if has_zoom: params.update({'zoom': zoom, 'lat': lat, 'lon': lng}) options.update({'zoom': zoom, 'lat': lat, 'lng': lng}) else: options.update(self._get_bounds(nb_layers)) map_name = self._send_map_template(layers, has_zoom=has_zoom) api_url = '{base_url}api/v1/map'.format(base_url=self.base_url) static_url = ('{api_url}/static/named/{map_name}' '/{width}/{height}.png?{params}').format( api_url=api_url, map_name=map_name, width=size[0], height=size[1], params=urlencode(params)) html = '<img src="{url}" />'.format(url=static_url) # TODO: write this as a private method if interactive: netloc = urlparse(self.base_url).netloc domain = 'carto.com' if netloc.endswith('.carto.com') else netloc def safe_quotes(text, escape_single_quotes=False): """htmlify string""" if isinstance(text, str): safe_text = text.replace('"', """) if escape_single_quotes: safe_text = safe_text.replace("'", "\'") return safe_text.replace('True', 'true') return text config = { 'user_name': self.username, 'maps_api_template': self.base_url[:-1], 'sql_api_template': self.base_url[:-1], 'tiler_protocol': 'https', 'tiler_domain': domain, 'tiler_port': '80', 'type': 'torque' if time_layer else 'namedmap', 'named_map': { 'name': map_name, 'params': { k: safe_quotes(v, escape_single_quotes=True) for k, v in dict_items(options) }, }, } map_options = { 'filter': ['http', 'mapnik', 'torque'], 'https': True, } if time_layer: config.update({ 'order': 1, 'options': { 'query': time_layer.query, 'user_name': self.username, 'tile_style': time_layer.torque_cartocss, } }) config['named_map'].update({ 'layers': [{ 'layer_name': 't', }], }) map_options.update({ 'time_slider': True, 'loop': True, }) bounds = [] if has_zoom else [[options['north'], options['east']], [options['south'], options['west']]] content = self._get_iframe_srcdoc(config=config, bounds=bounds, options=options, map_options=map_options) img_html = html html = ( '<iframe srcdoc="{content}" width={width} height={height}>' ' Preview image: {img_html}' '</iframe>' ).format(content=safe_quotes(content), width=size[0], height=size[1], img_html=img_html) return IPython.display.HTML(html) else: try: import matplotlib.image as mpi import matplotlib.pyplot as plt except ImportError: warn('Matplotlib not detected. Saving image directly to disk') raise NotImplementedError raw_data = mpi.imread(static_url) f = plt.gcf() if ax is None: ax = plt.gca() ax.imshow(raw_data) ax.axis('off') return ax
def test_cartocontext_write(self): """context.CartoContext.write normal usage""" from cartoframes.context import MAX_ROWS_LNGLAT cc = cartoframes.CartoContext(base_url=self.baseurl, api_key=self.apikey) data = { 'nums': list(range(100, 0, -1)), 'category': [random.choice('abcdefghijklmnop') for _ in range(100)], 'lat': [0.01 * i for i in range(100)], 'long': [-0.01 * i for i in range(100)] } schema = { 'nums': int, 'category': 'object', 'lat': float, 'long': float } df = pd.DataFrame(data).astype(schema) cc.write(df, self.test_write_table) # check if table exists resp = self.sql_client.send(''' SELECT * FROM {table} LIMIT 0 '''.format(table=self.test_write_table)) self.assertIsNotNone(resp) # check that table has same number of rows resp = self.sql_client.send(''' SELECT count(*) FROM {table}'''.format(table=self.test_write_table)) self.assertEqual(resp['rows'][0]['count'], len(df)) # should error for existing table with self.assertRaises(NameError): cc.write(df, self.test_read_table, overwrite=False) # overwrite table and create the_geom column cc.write(df, self.test_write_table, overwrite=True, lnglat=('long', 'lat')) resp = self.sql_client.send(''' SELECT count(*) AS num_rows, count(the_geom) AS num_geoms FROM {table} '''.format(table=self.test_write_table)) # number of geoms should equal number of rows self.assertEqual(resp['rows'][0]['num_rows'], resp['rows'][0]['num_geoms']) # test batch lnglat behavior n_rows = MAX_ROWS_LNGLAT + 1 df = pd.DataFrame({ 'latvals': [random.random() for r in range(n_rows)], 'lngvals': [random.random() for r in range(n_rows)] }) job = cc.write(df, self.test_write_lnglat_table, lnglat=('lngvals', 'latvals')) self.assertIsInstance(job, cartoframes.context.BatchJobStatus) # test batch writes n_rows = 550000 df = pd.DataFrame({'vals': [random.random() for r in range(n_rows)]}) cc.write(df, self.test_write_batch_table) resp = self.sql_client.send(''' SELECT count(*) AS num_rows FROM {table} '''.format(table=self.test_write_batch_table)) # number of rows same in dataframe and carto table self.assertEqual(n_rows, resp['rows'][0]['num_rows']) cols = self.sql_client.send(''' SELECT * FROM {table} LIMIT 1 '''.format(table=self.test_write_batch_table)) expected_schema = { 'vals': { 'type': 'number' }, 'the_geom': { 'type': 'geometry' }, 'the_geom_webmercator': { 'type': 'geometry' }, 'cartodb_id': { 'type': 'number' } } # table should be properly created # util columns + new column of type number self.assertDictEqual(cols['fields'], expected_schema) # test properly encoding df = pd.DataFrame({'vals': [1, 2], 'strings': ['a', 'ô']}) cc.write(df, self.test_write_table, overwrite=True) # check if table exists resp = self.sql_client.send(''' SELECT * FROM {table} LIMIT 0 '''.format(table=self.test_write_table)) self.assertIsNotNone(resp) cc.delete(self.test_write_table) df = pd.DataFrame({'vals': list('abcd'), 'ids': list('wxyz')}) df = df.astype({'vals': str, 'ids': str}) cc.write(df, self.test_write_table) schema = cc.sql_client.send('select ids, vals from {}'.format( self.test_write_table))['fields'] self.assertSetEqual(set([schema[c]['type'] for c in schema]), set(('string', ))) df = pd.DataFrame({ 'vals': list('abcd'), 'ids': list('wxyz'), 'nums': [1.2 * i for i in range(4)], 'boolvals': [ True, False, None, True, ], }) cc.write(df, self.test_write_table, overwrite=True, type_guessing='true') resp = cc.sql_client.send('SELECT * FROM {}'.format( self.test_write_table))['fields'] schema = {k: v['type'] for k, v in dict_items(resp)} ans = dict(vals='string', ids='string', nums='number', boolvals='boolean', the_geom='geometry', the_geom_webmercator='geometry', cartodb_id='number') self.assertDictEqual(schema, ans)
def test_cartocontext_write(self): """context.CartoContext.write normal usage""" cc = cartoframes.CartoContext(base_url=self.baseurl, api_key=self.apikey) data = {'nums': list(range(100, 0, -1)), 'category': [random.choice('abcdefghijklmnop') for _ in range(100)], 'lat': [0.01 * i for i in range(100)], 'long': [-0.01 * i for i in range(100)]} schema = {'nums': int, 'category': 'object', 'lat': float, 'long': float} df = pd.DataFrame(data).astype(schema) dataset = cc.write(df, self.test_write_table) self.test_write_table = dataset.table_name # check if table exists resp = self.sql_client.send(''' SELECT * FROM {table} LIMIT 0 '''.format(table=self.test_write_table)) self.assertIsNotNone(resp) # check that table has same number of rows resp = self.sql_client.send(''' SELECT count(*) FROM {table}'''.format(table=self.test_write_table)) self.assertEqual(resp['rows'][0]['count'], len(df)) # should error for existing table err_msg = ('Table with name {t} and schema {s} already exists in CARTO. Please choose a different `table_name`' 'or use if_exists="replace" to overwrite it').format(t=self.test_write_table, s='public') with self.assertRaises(CartoException, msg=err_msg): cc.write(df, self.test_read_table, overwrite=False) # overwrite table and create the_geom column cc.write(df, self.test_write_table, overwrite=True, lnglat=('long', 'lat')) resp = self.sql_client.send(''' SELECT count(*) AS num_rows, count(the_geom) AS num_geoms FROM {table} '''.format(table=self.test_write_table)) # number of geoms should equal number of rows self.assertEqual(resp['rows'][0]['num_rows'], resp['rows'][0]['num_geoms']) cc.delete(self.test_write_table) df = pd.DataFrame({'vals': list('abcd'), 'ids': list('wxyz')}) df = df.astype({'vals': str, 'ids': str}) cc.write(df, self.test_write_table, overwrite=True) schema = cc.sql_client.send('select ids, vals from {}'.format( self.test_write_table))['fields'] self.assertSetEqual(set([schema[c]['type'] for c in schema]), set(('string', ))) df = pd.DataFrame({'vals': list('abcd'), 'ids': list('wxyz'), 'nums': [1.2 * i for i in range(4)], 'boolvals': [True, False, None, True, ], }) df['boolvals'] = df['boolvals'].astype(bool) cc.write(df, self.test_write_table, overwrite=True) resp = cc.sql_client.send('SELECT * FROM {}'.format( self.test_write_table))['fields'] schema = {k: v['type'] for k, v in dict_items(resp)} ans = dict(vals='string', ids='string', nums='number', boolvals='boolean', the_geom='geometry', the_geom_webmercator='geometry', cartodb_id='number') self.assertDictEqual(schema, ans)