def select_upload_data(dataset_to_update, dataframe): try: # initialize an empty list to hold the features that we have already uploaded to mapbox existing = [] # initialize API session datasets = Datasets() os.environ['MAPBOX_ACCESS_TOKEN'] = 'sk.eyJ1IjoiZmlyZS1yaXNrLXNmIiwiYSI6ImNqZGw1dTlwdDA1aXMzM3FrbDZpZnpmczMifQ.JbUPsvx8384WHOAfA9Vy9w' # get features of the mapbox dataset by calling the API features_collection = datasets.list_features(dataset_to_update).json() # loop through the features of the mapbox dataset and save them to the list "existing" for i in range(0,len(features_collection['features'])): existing.append([ features_collection['features'][i]["id"], features_collection['features'][i]["properties"]["name"], ]) # convert "existing" to a dataframe existing = pd.DataFrame(existing,columns=["id","address"]) print("succesfully accessed the existing mapbox dataset") except: print("Error: could not access the mapbox dataset. Check that you are accessing your token correctly") try: # get rows in dataframe that are not yet uploaded new_rows = pd.merge(left = dataframe, right = existing, on = "address", how = "left") new_rows = new_rows[new_rows.id.isnull() == True] new_rows = new_rows.set_index(np.arange(0,len(new_rows))) # create a "start point" for new row ids # note: this was not a very smart way to creat ids, but I'm stuck with it for now start = max(existing["id"].astype("int64")) # create new random ids for these new uploads new_ids = random.sample(range(start,start*5),len(new_rows)) print("succesfully selected new upload data") except: print("there was a problem uploading the data") # make sure these new random ids are not already in the dataset try: [new_id not in existing["id"] for new_id in new_ids] except: "Error! Duplicate ids created" # return the new rows and new ids to upload return new_rows, new_ids
def create_dataset(self, name, description): datasets = Datasets() res = datasets.create(name=name, description=desciption) ds_id = None if res.ok: ds = res.json() ds_id = ds['id'] return ds_id
def check_upload_result(upload, dataset_to_update): try: # check that the dataset upload was successful datasets = Datasets() os.environ['MAPBOX_ACCESS_TOKEN'] = 'sk.eyJ1IjoiZmlyZS1yaXNrLXNmIiwiYSI6ImNqZGw1dTlwdDA1aXMzM3FrbDZpZnpmczMifQ.JbUPsvx8384WHOAfA9Vy9w' resp = datasets.list_features(dataset_to_update).json() assert len(resp['features']) == len(upload), "Only " + str(len(resp['features'])) + " features are included in the dataset" except: print("Error: unable to check the number of uploaded records. To manually check the execution: Mapbox > Studio > Datasets > Menu > View details")
def upload_dataset_features(self, dataset_id, dataset_filename): """ Upload a dataset to mapbox This is rather slow uploading one by one so use upload as tileset """ datasets = Datasets() with open(os.path.join(DATASETS_PATH, dataset_filename)) as src: features = json.load(src) for fid, feature in enumerate(features.get("features", [])): datasets.update_feature(dataset_id, fid, feature)
def upload_to_mapbox(upload, dataset_to_update): try: # initialize api datasets = Datasets() os.environ['MAPBOX_ACCESS_TOKEN'] = 'sk.eyJ1IjoiZmlyZS1yaXNrLXNmIiwiYSI6ImNqZGw1dTlwdDA1aXMzM3FrbDZpZnpmczMifQ.JbUPsvx8384WHOAfA9Vy9w' # upload features to new dataset for i in range(0,len(upload['features'])): feature = upload['features'][i] index = upload['features'][i]['id'] resp = datasets.update_feature(dataset = dataset_to_update, fid = index, feature = feature) except: print("Error: could not upload the new data to Mapbox. Is there a problem with the access token?")
def insert_or_update_features(dataset_id, features): ''' Takes a list of features (dictionaries formatted as features) and adds them to dataset. If the ID of the feature is already in the dataset, it will be updated. If it's new, it will be inserted. Returns list of updated features. Will raise error if response is not 200. ''' datasets = Datasets() loads = [] for feature in features: resp = datasets.update_feature(dataset_id, feature['id'], feature) if resp.status_code != 200: err = (f'Response was {resp.status_code} when trying to', f'update {feature} in dataset with ID {dataset_id}', f'{resp.json()["message"]}') raise RuntimeError(err) loads.append(resp) return loads
class Mapbox(): datasets = Datasets(access_token=ALL_AT) uploader = Uploader(access_token=ALL_AT) username = '******' @classmethod def datasets_list(cls, response): resp = cls.datasets.list() return HttpResponse(resp.json(), content_type=json_content_type) @classmethod def datasets_insert(cls, response, dataset_id): ## pokud je nějaká změna musí se updated_to_mapbox změnit na false mista = Misto.objects.filter(active=True, uploaded_to_mapbox=False) results = {} for misto in mista: feature = { 'id': str(misto.id), 'type': 'Feature', 'properties': { 'nazev': misto.nazev, 'popis': misto.popis }, 'geometry': { 'type': 'Point', 'coordinates': [misto.y, misto.x] } } resp = cls.datasets.update_feature(dataset_id, str(misto.id), feature).json() results[misto.id] = resp misto.uploaded_to_mapbox = True misto.save() return HttpResponse(dumps(results), content_type=json_content_type) @classmethod def upload_tileset_from_dataset(cls, request, tileset_id, dataset_id): # https://github.com/mapbox/mapbox-sdk-py/issues/152#issuecomment-311708422 # https://www.mapbox.com/api-documentation/?language=Python#create-an-upload uri = "mapbox://datasets/{username}/{dataset_id}".format( username=cls.username, dataset_id=dataset_id) res = cls.uploader.create(uri, tileset_id, name='Msta') return HttpResponse(dumps({ 'tileset_id': tileset_id, 'dataset_id': dataset_id, 'result': res.json() }), content_type=json_content_type)
def handle(self, *args, **options): datasets = Datasets(access_token=settings.MAPBOX_TOKEN) try: mapbox_dataset = MapBoxDataset.objects.all()[0] #check if a dataset was already created dataset_id = mapbox_dataset.dataset_id dataset = datasets.read_dataset(dataset_id).json() except IndexError: #it wasn't, let's do that dataset = datasets.create(name='os-schools', description='A listing of OpenStax Adoptions') dataset_decoded = ast.literal_eval(dataset.content.decode()) mapbox_dataset_created, _ = MapBoxDataset.objects.get_or_create(name=dataset_decoded["name"], dataset_id=dataset_decoded["id"]) dataset_id = mapbox_dataset_created.dataset_id #cool - we have a dataset, now let's fill it with school location data schools = School.objects.all() for school in schools: feature = { 'type': 'Feature', 'geometry': { 'type': "Point", 'coordinates': [float(school.long), float(school.lat)] }, 'properties': { 'name': school.name } } datasets.update_feature(dataset_id, school.pk, feature) self.stdout.write("fin")
from shapely.geometry import Polygon, Point, LineString import os os.environ[ 'MAPBOX_ACCESS_TOKEN'] = "pk.eyJ1IjoidmliaHUxOTk0NiIsImEiOiJjam01MjB1YjkwMDZrM3BsMHI4d21reXduIn0.j1yPdjaTrJ1i4Uc_dcT62g" import pandas as pd import numpy as np from mapbox import Datasets import matplotlib.pyplot as plt import json import sys import geopandas as gpd import descartes from pandas.io.json import json_normalize import pyvisgraph as vg datasets = Datasets() col = datasets.list_features("ck5gcnyor0rcw2nmdwaeaiko9").json() #attrs = datasets.read_dataset("ck6ub6amu0h6g2nlhtl42fguy").json() df = pd.read_json(json.dumps(col['features'])) #df.head() #nycc = json_normalize(df['properties']) nyc = json_normalize(df['geometry']) #len(nycc.index) #== len(nyc.index) #nyc.head() pp = [] for i in nyc.index: for ff in nyc['coordinates'][i]: pp.append(Polygon(ff))
load_dotenv() if len(sys.argv) != 3: print( 'Please supply a mode <create|publish> and tile set <plaetze|strassen|viertel|denkmaeler|gebaeude>.' ) sys.exit(1) mode = sys.argv[1] tileset = sys.argv[2] url = 'https://api.mapbox.com/tilesets/v1/' access_string = '?access_token=' + os.getenv("tileset_token") if 'source' in mode: dataset_id = os.getenv(tileset + '_dataset') datasets = Datasets(access_token=os.getenv("dataset_token")) collection = datasets.list_features(dataset_id).json() features = collection['features'] ld_features = '\n'.join(json.dumps(feature) for feature in features) files = {'file': (tileset + '.geojson', ld_features)} sources_url = url + 'sources/strassenlaerm/' + tileset + access_string if mode == 'create_source': r = requests.post(sources_url, files=files) elif mode == 'replace_source': r = requests.put(sources_url, files=files) else: print('Please use a valid mode.') sys.exit(1) elif mode == 'create_tileset': with open('recipe-' + tileset + '.json', 'r') as recipe_file: r = requests.post(url + 'strassenlaerm.' + tileset + access_string,
#!/usr/bin/env python3 # -*- coding: utf-8 -*- import sys import os import json from mapbox import Datasets from dotenv import load_dotenv load_dotenv() datasets = Datasets(access_token=os.getenv("dataset_token")) streets_dataset = os.getenv("strassen_dataset") squares_dataset = os.getenv("plaetze_dataset") existing_streets = datasets.list_features(streets_dataset).json()['features'] existing_squares = datasets.list_features(squares_dataset).json()['features'] existing_objects = existing_streets + existing_squares # print(existing_objects) def generate_geometry(street_square, parts, msg): def reverse_part(part_to_reverse): part_to_reverse['properties']['endet_bei_'], part_to_reverse['properties']['beginnt_be'] = \ part_to_reverse['properties']['beginnt_be'], part_to_reverse['properties']['endet_bei_'] part_to_reverse['geometry']['coordinates'][0].reverse() if street_square['properties']['type'] == 'street': # print(street_square['properties']['name']) parts_multi = [] # go through all parts belonging to an object to simplify geometry # for p in parts: # print(p['properties']['beginnt_be'] + ' -> ' + p['properties']['endet_bei_'] + ': ' + str(p['geometry']['coordinates']))
def create_dataset(name, description): ''' Creates a new empty dataset with name and description. Returns newly created dataset object. ''' datasets = Datasets() return datasets.create(name=name, description=description)
def get_feature(dataset_id, feature_id): ''' Takes the ID of a dataset and ID of a feature. Returns the feature in dict. ''' datasets = Datasets() return datasets.read_feature(dataset_id, feature_id).json()
def get_features(dataset_id): ''' Takes the ID of a dataset and returns a dict with all the features in the dataset ''' datasets = Datasets() return datasets.list_features(dataset_id).json()
def datasets(*args, **kwargs): return Datasets()
MAPBOX_ACCESS_TOKEN = "sk.eyJ1IjoicGFwcHVzc3AxIiwiYSI6ImNqYndrZ3RrMTI2eDIzM3BjaXFtY2gzdmcifQ.B2sQFFPWo5tBrvcsL9cDVQ" from mapbox import Datasets from flask import Flask, request from flask_sqlalchemy import SQLAlchemy from flask_heroku import Heroku from sqlalchemy import create_engine import requests datasets = Datasets(access_token=MAPBOX_ACCESS_TOKEN) app = Flask(__name__) heroku = Heroku(app) db = SQLAlchemy(app) #engine = create_engine('postgresql+psycopg2://shankoibito:pappussp@localhost/tiotsudatamap') app.config[ 'SQLALCHEMY_DATABASE_URI'] = 'postgres://shanbirsingh@toitsu:[email protected]:5432/postgres' #app.config['SQLALCHEMY_DATABASE_URI'] = 'postgres://*****:*****@ec2-54-227-250-33.compute-1.amazonaws.com:5432/d9p6o27d01ao21' app.config['SECRET_KEY'] = 'oh_so_secret' class tiotsu_users(db.Model): __tablename__ = "tiotsu_users" email = db.Column(db.String(120), nullable=False, primary_key=True) firstname = db.Column(db.String(100), nullable=False) yunk = db.Column(db.String(100)) level = db.Column(db.String(50), nullable=False) aura = db.Column(db.String(100)) houselevel = db.Column(db.String(50)) mylocation = db.Column(db.String(150), nullable=False, unique=True) help = db.Column(db.String(100))
from mapbox import Datasets import json datasets = Datasets( access_token= 'sk.eyJ1IjoibG9raXByJjamc4aGRpZ3oyM3BxMzNuNWIzaDdja2ZzIn0.oFqNkH9Mlyv3ExsGknvSGg' ) create_resp = datasets.create(name="Bay Area Zips", description="ZTCA zones for the Bay Area") listing_resp = datasets.list() dataset_id = [ds['id'] for ds in listing_resp.json()][0] data = json.load(open(r'ztca_bayarea.geojson')) for count, feature in enumerate(data['features'][:1]): resp = datasets.update_feature(dataset_id, count, feature)
import json from mapbox import Datasets access_token = '<api key with read, write, and list access for Datasets>' # Remove any datasets with the name "demo". This can be removed for wider use [ datasets.delete_dataset(d['id']) for d in datasets.list().json() if d['name'] == 'demo' ] # Create a new, empty dataset and get info about it datasets = Datasets(access_token=access_token) create_resp = datasets.create( name='demo', description='Demo dataset for Datasets and Uploads APIs') dataset_info = create_resp.json() print(dataset_info) # Add data to new dataset # Make sure there is no "id" key-value pair outside the "properties" object. Mapbox has pretty strict standards for GeoJSON. data = json.load(open(r'demo_pts.geojson')) for count, feature in enumerate(data['features']): print('{}, {}'.format(count, feature)) resp = datasets.update_feature(dataset_info['id'], count, feature) print(resp)
#!/usr/bin/env python3 import json import re from mapbox import Datasets datasets = Datasets() datasetID = 'ciuoufm6l01ch2yp4nzm3tjtb' for lat in range(-90, 100, 10): id = "lat%d" % (lat) feature = { "type": "Feature", "id": id, "geometry": { "type": "LineString", "coordinates": [[-180, lat], [180, lat]] }, "properties": { "name": "%d" % lat, "id": id } } resp = datasets.update_feature(datasetID, id, feature) print(id) for lon in range(-180, 190, 10): id = "lon%d" % (lon) feature = { "type": "Feature", "id": id,
from mapbox import Datasets import random as rand datasets = Datasets( access_token= "sk.eyJ1IjoiYW5hbnRqIiwiYSI6ImNqcm5pa3NjbDA2cW8zeW8zb25nOTczY2sifQ.QkSZBaZnn6KxwO-YpRtA8A" ) def getJSON(name): collections = datasets.list_features('cjrna6cc80dbo2wphvazyuf89').json() collections = collections['features'] out = -1 for col in collections: if (col['properties']['title'] == name): out = col break return out def update(name, value): data = getJSON(name) if (not (data == -1)): data['properties']['description'] = value update = datasets.update_feature('cjrna6cc80dbo2wphvazyuf89', data['id'], data).json() return update else: return None