def test_cloud_personal_url(): user1 = APIKeyAuthClient('https://user1.carto.com/a/b/c', API_KEY).username user2 = APIKeyAuthClient('https://www.user2.carto.com', API_KEY).username user3 = APIKeyAuthClient('http://www.user3.carto.com/a/b/c', API_KEY).username assert user1 == 'user1' assert user2 == 'user2' assert user3 == 'user3'
def test_api_key_auth_can_only_read_default_public_with_default_public(): client = APIKeyAuthClient(USER1_BASE_URL, DEFAULT_PUBLIC_API_KEY) response = client.send('/api/v3/api_keys', 'get') assert response.status_code == 200 api_keys = response.json() assert api_keys['count'] == 1 assert len(api_keys['result']) == 1 assert api_keys['result'][0]['name'] == 'Default public' assert api_keys['result'][0]['token'] == 'default_public'
def test_on_prem_url(): user1 = APIKeyAuthClient('https://carto.com/user/user1/a/b/c', API_KEY).username user2 = APIKeyAuthClient('https://www.carto.com/user/user2', API_KEY).username user3 = APIKeyAuthClient('http://www.carto.com/user/user3/a/b/c', API_KEY).username assert user1 == 'user1' assert user2 == 'user2' assert user3 == 'user3'
def test_user_agent(): expected_user_agent = _ClientIdentifier().get_user_agent() adapter = requests_mock.Adapter() session = requests.Session() # Using file:// cause urllib's urljoin (used in pyrestcli) # does not support a mock:// schema session.mount('file', adapter) adapter.register_uri('POST', 'file://test.carto.com/headers', request_headers={'User-Agent': expected_user_agent}) client = APIKeyAuthClient('file://test.carto.com', 'some_api_key', None, session) client.send('headers', 'post')
def __init__(self, base_url=None, api_key=None, session=None, verbose=0): self.api_key, self.base_url = _process_credentials(api_key, base_url) self.auth_client = APIKeyAuthClient(base_url=self.base_url, api_key=self.api_key, session=session) self.sql_client = SQLClient(self.auth_client) self.username = self.auth_client.username self.is_org = self._is_org_user() self._map_templates = {} self._srcdoc = None self._verbose = verbose
def initialize(self): if not self.api_url and self.user_name: self.api_url = "https://{}.carto.com/api/".format(self.user_name) elif not self.api_url and not self.user_name: raise Exception( 'Not enough data provided to initialize the client') if self.org_name: self.client = APIKeyAuthClient(self.api_url, self.api_key, self.org_name) else: self.client = APIKeyAuthClient(self.api_url, self.api_key) self.sql_client = SQLClient(self.client) self.batch_client = BatchSQLClient(self.client)
def get_auth_client(username=None, api_key=None, baseurl=None, cdb_client=None): """Instantiates a SQL Client from the CARTO Python SDK (v1.0.0) :param username: CARTO username :type username: string :param api_key: API key of CARTO user ``username`` :type api_key: string :param baseurl: Base URL for CARTO instance (usually suitable for on prem) :type baseurl: string :param cdb_client: CARTO Python SDK Authentication client :type cdb_client: object :returns: Authenticated SQL client with user credentials :rtype: sql auth object """ from carto.sql import SQLClient from carto.auth import APIKeyAuthClient if cdb_client: sql = SQLClient(cdb_client) elif username is not None and api_key is not None: baseurl = get_baseurl(username=username, baseurl=baseurl) auth_client = APIKeyAuthClient(baseurl, api_key) sql = SQLClient(auth_client) else: raise Exception("`username` and `api_key` or `cdb_client` has to be " "specified.") return sql
def setUp(self): if (os.environ.get('APIKEY') is None or os.environ.get('USERNAME') is None): try: creds = json.loads(open('test/secret.json').read()) self.apikey = creds['APIKEY'] self.username = creds['USERNAME'] except: # noqa warnings.warn('Skipping CartoContext tests. To test it, ' 'create a `secret.json` file in test/ by ' 'renaming `secret.json.sample` to `secret.json` ' 'and updating the credentials to match your ' 'environment.') self.apikey = None self.username = None else: self.apikey = os.environ['APIKEY'] self.username = os.environ['USERNAME'] if self.username and self.apikey: self.baseurl = 'https://{username}.carto.com/'.format( username=self.username) self.auth_client = APIKeyAuthClient(base_url=self.baseurl, api_key=self.apikey) self.sql_client = SQLClient(self.auth_client) # sets skip value WILL_SKIP = self.apikey is None or self.username is None # noqa: F841 has_mpl = 'mpl' if os.environ.get('MPLBACKEND') else 'nonmpl' pyver = sys.version[0:3].replace('.', '_') # for writing to carto self.test_write_lnglat_table = ( 'cartoframes_test_write_lnglat_table_{ver}_{mpl}'.format( ver=pyver, mpl=has_mpl))
def user(): """ Handy way for tests to have access to a user object :return: User instance that correspond to the test user """ return UserManager(APIKeyAuthClient(BASE_URL, API_KEY, ORGANIZATION)). \ get(USERNAME) if ORGANIZATION is not None else None
def _create_auth_client(credentials, public=False): return APIKeyAuthClient( base_url=credentials.base_url, api_key='default_public' if public else credentials.api_key, session=credentials.session, client_id='cartoframes_{}'.format(__version__), user_agent='cartoframes_{}'.format(__version__))
def setUp(self): if (os.environ.get('APIKEY') is None or os.environ.get('USERNAME') is None): try: creds = json.loads(open('test/secret.json').read()) self.apikey = creds['APIKEY'] self.username = creds['USERNAME'] except: # noqa: E722 warnings.warn("Skipping CartoContext tests. To test it, " "create a `secret.json` file in test/ by " "renaming `secret.json.sample` to `secret.json` " "and updating the credentials to match your " "environment.") self.apikey = None self.username = None else: self.apikey = os.environ['APIKEY'] self.username = os.environ['USERNAME'] self.user_url = self.user_url() if self.username and self.apikey: self.baseurl = self.user_url.format(username=self.username) self.auth_client = APIKeyAuthClient(base_url=self.baseurl, api_key=self.apikey) self.sql_client = SQLClient(self.auth_client) # sets skip value WILL_SKIP = self.apikey is None or self.username is None # noqa: F841 self.points = 'tweets_obama' self.polys = 'nat' self.local = 'cb_2013_us_csa_500k'
def update_asset_on_carto(asset_dict, fields): auth_client = APIKeyAuthClient(api_key=CARTO_API_KEY, base_url=USR_BASE_URL) sql = SQLClient(auth_client) #values_tuple_strings = [make_values_tuple_string_from_model(r, fields) for r in [asset]] # OR POSSIBLY #values_tuple_strings = [make_values_tuple_string_from_model(asset, fields)] #q = f"UPDATE {TABLE_NAME} SET {values_tuple_strings} WHERE asset_id = {asset.id};" #values_tuple_strings = [values_string_from_model(asset, fields)] #q = f"UPDATE {TABLE_NAME} SET ({', '.join(fields + ['the_geom', 'the_geom_webmercator'])}) " \ # f"VALUES {', '.join(map(lambda x: x + 1, values_tuple_strings))};" # This is throwing an # error, and it's really not clear why it's trying to map a math function over strings. # Let's ignore the the_geom* fields for now and do the update the simple way: # Single updates can be done like this: # UPDATE election_results SET votes=52, pro=24 WHERE county_id = 1; other_fields = copy.deepcopy(fields) other_fields.remove('id') q = f"UPDATE {TABLE_NAME} SET {set_string_from_model(asset_dict, other_fields)} WHERE id = {asset_dict['asset'].id};" assert len(q) < 16384 print(q) results = sql.send(q)
def __init__(self, user, api_key, options={}): super().__init__(options) self.do_post = options.get('do_post', False) self.parse_json = options.get('parse_json', True) self.format = options.get('format', 'json') self.base_url_option = options.get('base_url', '') self.api_version = options.get('api_version', self.DEFAULT_API_VERSION) self.batch = options.get('batch', False) self.user = user self.api_key = api_key self.base_url = self._generate_base_url(user, self.base_url_option) # Carto Context for DataFrame handling self._carto_context = None # Carto client for COPYs self._copy_client = None self._auth_client = APIKeyAuthClient(api_key=api_key, base_url=self.base_url) self._sql_client = SQLClient(self._auth_client, api_version=self.api_version) self._batch_client = None if self.batch: self._batch_client = BatchSQLClient(self._auth_client)
def fix_carto_geofields(asset_id=None): auth_client = APIKeyAuthClient(api_key=CARTO_API_KEY, base_url=USR_BASE_URL) sql = SQLClient(auth_client) # Now the problem with pushing this data through SQL calls is that Carto does not rerun the # processes that add values for the_geom and the_geom_webmercator. So it kind of seems like # we have to do this ourselves as documented at # https://gis.stackexchange.com/a/201908 q = f"UPDATE {TABLE_NAME} SET the_geom = ST_SetSRID(st_makepoint(longitude, latitude),4326)" if asset_id is not None: q += f" WHERE id = {asset_id}" # This can significantly speed up Carto geofield updates # when saving a single model instance. # This works because 'longitude' and 'latitude' are the names of the corresponding fields in the CSV file. results1 = sql.send(q) # This takes 12 seconds to run for 100,000 rows. # Exporting the data immediately after this is run oddly leads to the same CSV file as exporting before # it is run, but waiting a minute and exporting again gives something with the_geom values in the same # rows as the table on the Carto site. Basically, the exported CSV file can lag the view on the Carto # web site by a minute or two. q = f"SELECT ST_Transform(ST_SetSRID(st_makepoint(longitude, latitude),4326),3857) as the_geom_webmercator FROM {TABLE_NAME}" results2 = sql.send(q) # This one ran much faster. # One improvement is that you can replace ST_SetSRID(st_makepoint(lon, lat)) with CDB_LatLng(lat, lon) # though I don't know if it leads to any performance improvement. print( f"Tried to add values for the the_geom and the_geom_webmercator fields in {TABLE_NAME}. The requests completed in {results1['time']} s and {results2['time']} s." )
def api_key_auth_client_usr(): """ Returns a API key authentication client that can be used to send authenticated test requests to CARTO :return: APIKeyAuthClient instance """ return APIKeyAuthClient(USR_BASE_URL, API_KEY)
def setUp(self): if (os.environ.get('APIKEY') is None or os.environ.get('USERNAME') is None): try: creds = json.loads(open('test/secret.json').read()) self.apikey = creds['APIKEY'] self.username = creds['USERNAME'] except: warnings.warn("Skipping CartoContext tests. To test it, " "create a `secret.json` file in test/ by " "renaming `secret.json.sample` to `secret.json` " "and updating the credentials to match your " "environment.") self.apikey = None self.username = None else: self.apikey = os.environ['APIKEY'] self.username = os.environ['USERNAME'] if self.username and self.apikey: self.baseurl = 'https://{username}.carto.com/'.format( username=self.username) self.auth_client = APIKeyAuthClient(base_url=self.baseurl, api_key=self.apikey) self.sql_client = SQLClient(self.auth_client) # sets skip value WILL_SKIP = self.apikey is None or self.username is None # table naming info has_mpl = 'mpl' if os.environ.get('MPLBACKEND') else 'nonmpl' pyver = sys.version[0:3].replace('.', '_') # test tables self.test_read_table = 'cb_2013_us_csa_500k' self.valid_columns = set([ 'affgeoid', 'aland', 'awater', 'created_at', 'csafp', 'geoid', 'lsad', 'name', 'the_geom', 'updated_at' ]) # for writing to carto self.test_write_table = 'cartoframes_test_table_{ver}_{mpl}'.format( ver=pyver, mpl=has_mpl) # for batch writing to carto self.test_write_batch_table = ( 'cartoframes_test_batch_table_{ver}_{mpl}'.format(ver=pyver, mpl=has_mpl)) self.test_write_lnglat_table = ( 'cartoframes_test_write_lnglat_table_{ver}_{mpl}'.format( ver=pyver, mpl=has_mpl)) # for queries self.test_query_table = ('cartoframes_test_query_' 'table_{ver}_{mpl}'.format(ver=pyver, mpl=has_mpl)) self.test_delete_table = ('cartoframes_test_delete_' 'table_{ver}_{mpl}').format(ver=pyver, mpl=has_mpl)
def get_carto_client(api_key, base_url): """ Returns APIKeyAuthClient object for CartoDB Python SDK. Parameters: - api_key: string - base_url: string """ return APIKeyAuthClient(api_key=api_key, base_url=base_url)
def wrong_onprem_auth_client(): """ Returns an authentication client that can be used to send authenticated test requests to CARTO onpremises :return: NonVerifiedAPIKeyAuthClient instance """ if ONPREM_API_KEY is None: return None return APIKeyAuthClient(ONPREMISES_USR_BASE_URL, ONPREM_API_KEY, ONPREM_ORGANIZATION)
def get_api_key_auth_client(self): if not self._api_key_auth_client: self._api_key_auth_client = APIKeyAuthClient( base_url=self._base_url, api_key=self.api_key, session=self.session, client_id='cartoframes_{}'.format(__version__), user_agent='cartoframes_{}'.format(__version__)) return self._api_key_auth_client
def get_carto_asset_ids(id_to_check=None): auth_client = APIKeyAuthClient(api_key=CARTO_API_KEY, base_url=USR_BASE_URL) sql = SQLClient(auth_client) if id_to_check is None: results = sql.send(f"SELECT id FROM {TABLE_NAME}") else: results = sql.send( f"SELECT id FROM {TABLE_NAME} WHERE id = {id_to_check}") ids = [r['id'] for r in results['rows']] return ids
def delete_map(map_name): auth_client = APIKeyAuthClient(CARTO_BASE_URL, CARTO_API_KEY) named_map_manager = NamedMapManager(auth_client) try: named_map = named_map_manager.get(map_name) if named_map is not None: named_map.client = auth_client named_map.delete() except Exception as e: #ignore print(e)
def create_map(iso, lon, lat): creds = Credentials(username=CARTO_USER_NAME, api_key=CARTO_API_KEY) set_default_credentials(creds) dataset_name = 'onekmiso' dataset_name = dataset_name + str(int(round(time.time() * 1000))) to_carto(iso, dataset_name, if_exists='replace', log_enabled=True) auth_client = APIKeyAuthClient(CARTO_BASE_URL, CARTO_API_KEY) FACTOR = DPI / 72.0 map_name = 'tpl_' + dataset_name create_named_map(auth_client, CARTO_USER_NAME, dataset_name, map_name, FACTOR, lon, lat) return map_name
def __init__(self): self.carto_api_key = os.environ['API_KEY'] self.carto_account = os.environ['ACCOUNT'] USR_BASE_URL = "https://{user}.carto.com/".format( user=self.carto_account) self.auth_client = APIKeyAuthClient(api_key=self.carto_api_key, base_url=USR_BASE_URL) #this mimics the carto docs, leave it this way self.sql = SQLClient(self.auth_client) self.dataset_manager = DatasetManager(self.auth_client)
def __init__(self, CARTO_USER, CARTO_API_URL, CARTO_ORG, CARTO_API_KEY, USER_QUOTA): self.CARTO_USER = CARTO_USER self.CARTO_ORG = CARTO_ORG self.USER_QUOTA = USER_QUOTA ### CARTO clients auth_client = APIKeyAuthClient(CARTO_API_URL, CARTO_API_KEY, CARTO_ORG) self.sql = SQLClient(auth_client) self.vm = VisualizationManager(auth_client) self.dm = DatasetManager(auth_client) ### logger, variables and CARTO clients self.logger = logging.getLogger('carto_report') self.logger.addHandler(logging.NullHandler())
def upload_to_carto(csv_loc): print('Uploading processed data to Carto.') #set up carto authentication using local variables for username (CARTO_WRI_RW_USER) and API key (CARTO_WRI_RW_KEY) auth_client = APIKeyAuthClient(api_key=os.getenv('CARTO_WRI_RW_KEY'), base_url="https://{user}.carto.com/".format( user=os.getenv('CARTO_WRI_RW_USER'))) #set up dataset manager with authentication dataset_manager = DatasetManager(auth_client) #upload dataset to carto dataset = dataset_manager.create(csv_loc) print('Carto table created: {}'.format( os.path.basename(csv_loc).split('.')[0])) #set dataset privacy to 'Public with link' dataset.privacy = 'LINK' dataset.save() print('Privacy set to public with link.')
def index(): form = MainForm() if form.validate_on_submit(): auth_client = APIKeyAuthClient(form.carto_base_url.data, form.carto_api_key.data) if form.incident_type.data == "alerts": incidents = AlertProcessor(auth_client, username=form.db_username.data, password=form.db_password.data, host=form.db_host.data) elif form.incident_type.data == "jams": incidents = JamProcessor(auth_client, username=form.db_username.data, password=form.db_password.data, host=form.db_host.data) else: flash("Unknown incident type: {incident_type}".format(incident_type=form.incident_type.data), "error") return render_template("index.html", form=form) with io.StringIO() as csv_out: try: incidents.get_values(csv_out, pub_utc_date__gt=form.from_timestamp.data, pub_utc_date__lt=form.to_timestamp.data) except psycopg2.OperationalError as e: form.db_host.errors = True flash(e, "error") return render_template("index.html", form=form) else: csv_out.seek(0) with io.BytesIO(bytes(csv_out.read(), "utf-8")) as csv_in: table_name = form.carto_table_name.data or get_table_name() try: incidents.create_table(table_name=table_name, cartodbfy=True) except CartoException as e: form.carto_base_url.errors = True flash(e, "error") return render_template("index.html", form=form) try: incidents.append_data(csv_in, table_name=table_name) except CartoException as e: flash(e, "error") return render_template("index.html", form=form) flash("Data has been saved in CARTO ({table_name})".format(table_name=table_name), "success") else: if form.errors: flash(form.errors, "error") return render_template("index.html", form=form)
def conn(self): if self._conn is None: self.logger.info('Making connection to Carto {} account...'.format( self.user)) try: api_key = self.api_key base_url = USR_BASE_URL.format(user=self.user) auth_client = APIKeyAuthClient(api_key=api_key, base_url=base_url) conn = SQLClient(auth_client) self._conn = conn self.logger.info('Connected to Carto.\n') except CartoException as e: self.logger.error( 'Failed making connection to Carto {} account...'.format( self.user)) raise e return self._conn
def upload_to_carto(file, privacy): ''' Upload tables to Carto INPUT file: location of file on local computer that you want to upload (string) ''' # set up carto authentication using local variables for username (CARTO_WRI_RW_USER) and API key (CARTO_WRI_RW_KEY) auth_client = APIKeyAuthClient(api_key=os.getenv('CARTO_WRI_RW_KEY'), base_url="https://{user}.carto.com/".format( user=os.getenv('CARTO_WRI_RW_USER'))) # set up dataset manager with authentication dataset_manager = DatasetManager(auth_client) # upload dataset to carto dataset = dataset_manager.create(file) logger.info('Carto table created: {}'.format( os.path.basename(file).split('.')[0])) # set dataset privacy to 'Public with link' dataset.privacy = privacy logger.info('Privacy set to public with link.') dataset.save()
def __init__(self, config_path): """ Initializes the instance Args: config_path: the path to the configuration file """ self.logger = logging.getLogger('sb') """The instance logger""" self.config = self.load_config(config_path) """Configuration dict""" self.logger.setLevel(self.config['log_level']) self.logger.debug('Configuration processed') creds = self.config["carto_account"] self.logger.debug(creds) self.auth_client = APIKeyAuthClient(creds["api_url"], creds["api_key"], creds["organization"]) """API key CARTO authentication client"""
def dumpToCarto(eleList, table_name): # I am using my CARTO account USERNAME = "******" USR_BASE_URL = "https://{user}.carto.com/".format(user=USERNAME) auth_client = APIKeyAuthClient( api_key="53bb19efc968a08f7bdc2c1ffc29c31659240b39", base_url=USR_BASE_URL) sql = SQLClient(auth_client) table_name = 'strava_segments_' + table_name for segment in eleList: try: query = "UPDATE {table} SET cartodb_id={id}, the_geom=ST_SetSRID(ST_MakePoint({long}, {lat}),4326), name='{name}', value={value}, date=now() WHERE cartodb_id={id}". \ format(table=table_name,id=segment[0],long=segment[7],lat=segment[8],name=segment[1],value=segment[2]) logger.info(query) sql.send(query) except CartoException as e: logger.error(e)
def test_client_id_in_requests(): expected_client_id = _ClientIdentifier().get_client_identifier() client = APIKeyAuthClient('https://test.carto.com', 'some_api_key') http_method, requests_args = client.prepare_send('post') assert requests_args['params']['client'] == expected_client_id
def test_api_key_auth_client_me_endpoint(): client = APIKeyAuthClient(USER1_BASE_URL, API_KEY) username = client.send('/api/v3/me', 'get').json()['config']['user_name'] assert username == USER1_USERNAME