def test_delete(): job = json.loads(upload_response_body) responses.add( responses.DELETE, 'https://api.mapbox.com/uploads/v1/{0}/{1}?access_token={2}'.format(username, job['id'], access_token), match_querystring=True, body=None, status=204, content_type='application/json') res = mapbox.Uploader(access_token=access_token).delete(job) assert res.status_code == 204 res = mapbox.Uploader(access_token=access_token).delete(job['id']) assert res.status_code == 204
def test_status(): job = json.loads(upload_response_body) responses.add( responses.GET, 'https://api.mapbox.com/uploads/v1/{0}/{1}?access_token={2}'.format(username, job['id'], access_token), match_querystring=True, body=upload_response_body, status=200, content_type='application/json') res = mapbox.Uploader(access_token=access_token).status(job) assert res.status_code == 200 res = mapbox.Uploader(access_token=access_token).status(job['id']) assert res.status_code == 200 status = res.json() assert job == status
def test_upload_tileset_validation_specialchar(): with pytest.raises(mapbox.errors.ValidationError): with open('tests/moors.json', 'rb') as src: # limited to a-z0-9-_ chars mapbox.Uploader(access_token=access_token).upload(src, 'username.&#!', name='test1')
def test_create_name(): upload_response_body = """ {"progress": 0, "modified": "date.test", "error": null, "tileset": "testuser.test1", "complete": false, "owner": "testuser", "created": "date.test", "id": "id.test", "name": "testname"}""" def request_callback(request): payload = json.loads(request.body.decode()) assert payload['name'] == "testname" return (201, {}, upload_response_body) responses.add_callback( responses.POST, 'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format( username, access_token), match_querystring=True, callback=request_callback) res = mapbox.Uploader(access_token=access_token).create( 'http://example.com/test.json', 'testuser.test1', name="testname") assert res.status_code == 201 job = res.json() assert job['name'] == "testname"
def test_stage(monkeypatch): monkeypatch.setattr(mapbox.services.uploads, 'boto3_session', MockSession) # Credentials query_body = """ {{"key": "_pending/{username}/key.test", "accessKeyId": "ak.test", "bucket": "tilestream-tilesets-production", "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test", "secretAccessKey": "sak.test", "sessionToken": "st.test"}}""".format(username=username) responses.add( responses.POST, 'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token={1}'. format(username, access_token), match_querystring=True, body=query_body, status=200, content_type='application/json') with open('tests/moors.json', 'rb') as src: stage_url = mapbox.Uploader(access_token=access_token).stage(src) assert stage_url.startswith( "https://tilestream-tilesets-production.s3.amazonaws.com/_pending")
def test_big_stage(tmpdir, monkeypatch): """Files larger than 1M are multipart uploaded.""" monkeypatch.setattr(mapbox.services.uploads, 'boto3_session', MockSession) # Credentials query_body = """ {{"key": "_pending/{username}/key.test", "accessKeyId": "ak.test", "bucket": "tilestream-tilesets-production", "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test", "secretAccessKey": "sak.test", "sessionToken": "st.test"}}""".format(username=username) responses.add( responses.POST, 'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token={1}'. format(username, access_token), match_querystring=True, body=query_body, status=200, content_type='application/json') # Make a temp file larger than 1MB. bigfile = tmpdir.join('big.txt') bigfile.write(','.join(('num' for num in range(1000000)))) assert bigfile.size() > 1000000 with bigfile.open(mode='rb') as src: stage_url = mapbox.Uploader(access_token=access_token).stage(src) assert stage_url.startswith( "https://tilestream-tilesets-production.s3.amazonaws.com/_pending")
def test_upload_tileset_validation(): with pytest.raises(mapbox.errors.ValidationError): with open('tests/moors.json', 'rb') as src: mapbox.Uploader(access_token=access_token).upload(src, 'a' * 65, name='test1', patch=True)
def test_get_credentials(): query_body = """ {{"key": "_pending/{username}/key.test", "accessKeyId": "ak.test", "bucket": "tilestream-tilesets-production", "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test", "secretAccessKey": "sak.test", "sessionToken": "st.test"}}""".format(username=username) responses.add( responses.POST, 'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token={1}'. format(username, access_token), match_querystring=True, body=query_body, status=200, content_type='application/json') res = mapbox.Uploader(access_token=access_token)._get_credentials() assert res.status_code == 200 creds = res.json() assert username in creds['url'] for k in [ 'key', 'bucket', 'url', 'accessKeyId', 'secretAccessKey', 'sessionToken' ]: assert k in creds.keys()
def create_tileset(ctx, dataset, tileset, name): """Create a vector tileset from a dataset. $ mapbox datasets create-tileset dataset-id username.data Note that the tileset must start with your username and the dataset must be one that you own. To view processing status, visit https://www.mapbox.com/data/. You may not generate another tilesets from the same dataset until the first processing job has completed. All endpoints require authentication. An access token with `uploads:write` scope is required, see `mapbox --help`. """ access_token = (ctx.obj and ctx.obj.get('access_token')) or None service = mapbox.Uploader(access_token=access_token) uri = "mapbox://datasets/{username}/{dataset}".format( username=tileset.split('.')[0], dataset=dataset) res = service.create(uri, tileset, name) if res.status_code == 201: click.echo(res.text) else: raise MapboxCLIException(res.text.strip())
def test_upload_error(monkeypatch): """Upload a file and create a tileset, fails with 409""" monkeypatch.setattr(mapbox.services.uploads, 'boto3_session', MockSession) # Credentials query_body = """ {{"key": "_pending/{username}/key.test", "accessKeyId": "ak.test", "bucket": "tilestream-tilesets-production", "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test", "secretAccessKey": "sak.test", "sessionToken": "st.test"}}""".format(username=username) responses.add( responses.GET, 'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token={1}'.format(username, access_token), match_querystring=True, body=query_body, status=200, content_type='application/json') responses.add( responses.POST, 'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format(username, access_token), match_querystring=True, body="", status=409, content_type='application/json') with open('tests/moors.json', 'r') as src: res = mapbox.Uploader(access_token=access_token).upload(src, 'test1') assert res.status_code == 409
def test_upload_tileset_validation(): with pytest.raises(mapbox.errors.ValidationError): with open('tests/moors.json', 'rb') as src: # limited to 32 chars, try 40 mapbox.Uploader(access_token=access_token).upload(src, 'username.' + 'aA' * 20, name='test1')
def upload(ctx, args, name): """Upload data to Mapbox accounts. All endpoints require authentication. Uploaded data lands at https://www.mapbox.com/data/ and can be used in new or existing projects. You can specify the input file and tileset id $ mapbox upload mydata.geojson username.data Or specify just the tileset id and take an input file on stdin $ cat mydata.geojson | mapbox upload username.data The --name option defines the title as it appears in Studio and defaults to the last part of the tileset id, e.g. "data" Note that the tileset must start with your username. An access token with upload scope is required, see `mapbox --help`. """ access_token = (ctx.obj and ctx.obj.get('access_token')) or None service = mapbox.Uploader(access_token=access_token) if len(args) == 1: # Tileset specified, file from stdin click.echo("Reading data from stdin (Hit Ctl-C to cancel) ...", err=True) infile = BytesIO(click.File("rb")("-").read()) tileset = args[0] elif len(args) == 2: # Infile and Tileset are specified try: infile = click.File("rb")(args[0]) except click.ClickException: raise click.UsageError( "Could not open file: {0} " "(check order of command arguments: INFILE TILESET)".format( args[0])) tileset = args[1] else: raise click.UsageError( "Must provide either one argument (TILESET) or two (INFILE TILESET)" ) if name is None: name = tileset.split(".")[-1] try: res = service.upload(infile, tileset, name) except (mapbox.errors.ValidationError, IOError) as exc: raise click.BadParameter(str(exc)) if res.status_code == 201: click.echo(res.text) else: raise MapboxCLIException(res.text.strip())
def test_upload_tileset_validation_username(): # even with 60 chars, the addition of the # testuser username puts it over 64 chars with pytest.raises(mapbox.errors.ValidationError): with open('tests/moors.json', 'rb') as src: mapbox.Uploader(access_token=access_token).upload(src, 'a' * 60, name='test1', patch=True)
def test_create(): responses.add( responses.POST, 'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format(username, access_token), match_querystring=True, body=upload_response_body, status=201, content_type='application/json') res = mapbox.Uploader(access_token=access_token).create( 'http://example.com/test.json', 'test1') # without username prefix assert res.status_code == 201 job = res.json() assert job['tileset'] == "{0}.test1".format(username) res2 = mapbox.Uploader(access_token=access_token).create( 'http://example.com/test.json', 'testuser.test1') # also takes full tileset assert res2.status_code == 201 job = res2.json() assert job['tileset'] == "{0}.test1".format(username)
def upload(filename, upload_id): uploader = mapbox.Uploader() def attempt(): return attempt_upload(uploader, filename, upload_id) response = retry(attempt, retries=5, pause=5, succeeded=succeeded) if response.status_code != 201: raise MapboxError(response) return response
def test_list(): responses.add( responses.GET, 'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format(username, access_token), match_querystring=True, body="[{0}]".format(upload_response_body), status=200, content_type='application/json') res = mapbox.Uploader(access_token=access_token).list() assert res.status_code == 200 uploads = res.json() assert len(uploads) == 1 assert json.loads(upload_response_body) in uploads
def test_upload_patch(monkeypatch): """Upload a file and create a tileset in patch mode""" monkeypatch.setattr(mapbox.services.uploads, 'boto3_session', MockSession) def ensure_patch(request): payload = json.loads(request.body.decode()) assert payload['patch'] headers = {} return (201, headers, upload_response_body) # Credentials query_body = """ {{"key": "_pending/{username}/key.test", "accessKeyId": "ak.test", "bucket": "tilestream-tilesets-production", "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test", "secretAccessKey": "sak.test", "sessionToken": "st.test"}}""".format(username=username) responses.add( responses.POST, 'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token={1}'. format(username, access_token), match_querystring=True, body=query_body, status=200, content_type='application/json') responses.add_callback( responses.POST, 'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format( username, access_token), callback=ensure_patch, match_querystring=True, content_type='application/json') with open('tests/moors.json', 'rb') as src: res = mapbox.Uploader(access_token=access_token).upload( src, 'testuser.test1', name='test1', patch=True) assert res.status_code == 201 job = res.json() assert job['tileset'] == "{0}.test1".format(username)
def test_upload(monkeypatch): """Upload a file and create a tileset""" monkeypatch.setattr(mapbox.services.uploads, 'boto3_session', MockSession) # Credentials query_body = """ {{"key": "_pending/{username}/key.test", "accessKeyId": "ak.test", "bucket": "tilestream-tilesets-production", "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test", "secretAccessKey": "sak.test", "sessionToken": "st.test"}}""".format(username=username) responses.add( responses.POST, 'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token={1}'. format(username, access_token), match_querystring=True, body=query_body, status=200, content_type='application/json') responses.add( responses.POST, 'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format( username, access_token), match_querystring=True, body=upload_response_body, status=201, content_type='application/json') def print_cb(num_bytes): print("{0} bytes uploaded".format(num_bytes)) with open('tests/moors.json', 'rb') as src: res = mapbox.Uploader(access_token=access_token).upload( src, 'test1', callback=print_cb) assert res.status_code == 201 job = res.json() assert job['tileset'] == "{0}.test1".format(username)
def run(self): wx.PostEvent(self._notify_window, ResultEvent('---STARTED---\n')) self.log.write(self.st + '---STARTED---\n') wx.PostEvent(self._notify_window, ResultEvent('Path: ' + self.pfad + '\n')) self.log.write(self.st + 'Path: ' + self.pfad + '\n') wx.PostEvent(self._notify_window, ResultEvent('Dronename: ' + self.dronename + '\n')) self.log.write(self.st + 'Dronename: ' + self.dronename + '\n') wx.PostEvent(self._notify_window, ResultEvent('Datatype: ' + self.datatype + '\n')) self.log.write(self.st + 'Datatype: ' + self.datatype + '\n') wx.PostEvent( self._notify_window, ResultEvent('Starting Tileset: ' + str(self.starttileset) + '\n')) self.log.write(self.st + 'Starting Tileset: ' + str(self.starttileset) + '\n') #j = 1 while True: if os.path.isfile(self.pfad + '/' + self.dronename + str(self.starttileset) + self.datatype) == True: wx.PostEvent( self._notify_window, ResultEvent('Data ' + self.dronename + str(self.starttileset) + ' found! Preparing to upload:\n')) self.log.write(self.st + 'Data ' + self.dronename + str(self.starttileset) + ' found! Preparing to upload:\n') res = mapbox.Uploader( access_token=self.access_token)._get_credentials() with open( self.pfad + '/' + self.dronename + str(self.starttileset) + self.datatype, 'rb') as src: stage_url = mapbox.Uploader( access_token=self.access_token).stage(src) wx.PostEvent( self._notify_window, ResultEvent('Data ' + self.dronename + str(self.starttileset) + ' successfully staged. Start upload:\n')) self.log.write(self.st + 'Data ' + self.dronename + str(self.starttileset) + ' successfully staged. Start upload:\n') def print_cb(num_bytes): wx.PostEvent( self._notify_window, ResultEvent('{0} bytes uploaded\n'.format(num_bytes))) self.log.write(self.st + '{0} bytes uploaded\n'.format(num_bytes)) with open( self.pfad + '/' + self.dronename + str(self.starttileset) + self.datatype, 'rb') as src: res = mapbox.Uploader( access_token=self.access_token).upload( src, self.dronename + str(self.starttileset), callback=print_cb) wx.PostEvent( self._notify_window, ResultEvent( 'Data ' + self.dronename + str(self.starttileset) + ' successfully uploaded. Search for next:\n')) self.log.write( self.st + 'Data ' + self.dronename + str(self.starttileset) + ' successfully uploaded. Search for next:\n') self.starttileset += 1 time.sleep(3) else: wx.PostEvent( self._notify_window, ResultEvent('Data ' + self.dronename + str(self.starttileset) + ' not found. Sleep 3 seconds:\n')) self.log.write(self.st + 'Data ' + self.dronename + str(self.starttileset) + ' not found. Sleep 3 seconds:\n') time.sleep(3) if self._want_abort: wx.PostEvent(self._notify_window, ResultEvent('---STOPPED---\n')) self.log.write(self.st + '---STOPPED---\n') wx.PostEvent(self._notify_window, ResultEvent(None)) return
def test_invalid_fileobj(): """Must be file object, not path""" with pytest.raises(mapbox.errors.InvalidFileError): mapbox.Uploader(access_token=access_token).upload( 'tests/moors.json', 'test1')
def test_class_attrs(): """Get expected class attr values""" serv = mapbox.Uploader() assert serv.api_name == 'uploads' assert serv.api_version == 'v1'
def test_create_tileset_validation(): # even with 60 chars, the addition of the username puts it over 64 chars with pytest.raises(mapbox.errors.ValidationError): mapbox.Uploader(access_token=access_token).create( 'http://example.com/test.json', 'a' * 60)
geometry = [Point(xy) for xy in zip(ottawa['Longitude'], ottawa['Latitude'])] ottawa = gp.GeoDataFrame(ottawa, geometry=geometry) ottawa.crs = {'init': 'epsg:4326'} ottawa = ottawa.to_crs(gdb.crs) gdb = gdb.loc[(gdb['is_active'] == 1) & (gdb['aoi_code'] == 'QC_LowerOttawa')] ottawa_flood = gp.sjoin(ottawa, gdb, how='inner', op='within') ottawa_flood = ottawa_flood.loc[:, [ 'NodeID', 'StreetNumber', 'StreetName', 'geometry' ]] ottawa_flood = ottawa_flood.to_crs({'init': 'epsg:4326'}) ottawa_flood.to_file('ottawa_flood.geojson', driver='GeoJSON', encoding="utf-8") gdb = gp.overlay(gdb, gdb, how='union') gdb.geometry = gdb.geometry.simplify(10) gdb = gdb.explode() gdb = gdb.to_crs({'init': 'epsg:4326'}) gdb.to_file('extent.geojson', driver='GeoJSON', encoding="utf-8") u = mb.Uploader(access_token=sk) url = u.stage(open('ottawa_flood.geojson', 'rb')) job = u.create(url, 'ottawa_flood').json() url = u.stage(open('extent.geojson', 'rb')) job = u.create(url, 'extent').json()
# In[16]: print(output_file_path_tiles) # In[20]: command = "tippecanoe -o {} -Z {} -z {} {}.geojson".format( output_file_path_tiles, MIN_ZOOM_LIMIT, MAX_ZOOM_LIMIT, output_file_path) # In[23]: result = subprocess.check_output(command, shell=True) # In[24]: service = mapbox.Uploader() # In[25]: mapid = output_dataset_name # In[26]: print(mapid) # In[27]: with open(output_file_path_tiles, 'rb') as src: upload_resp = service.upload(src, mapid) # In[28]:
def invoke(self): # Token token = self.get_input_string_port('token') os.environ['MAPBOX_ACCESS_TOKEN'] = token # Tileset name tileset_name = self.get_input_string_port('tileset_name') # Timeout (in seconds) timeout = int(self.get_input_string_port('timeout', default='600')) # The following string inputs apply only to vector tiles # Min zoom min_zoom = self.get_input_string_port('min_zoom', default='0') # Max zoom max_zoom = self.get_input_string_port('max_zoom', default='14') # Properties to be excluded excluded_properties = self.get_input_string_port('excluded_properties', default='').split(',') # Filtering criteria criteria = self.get_input_string_port('criteria', default='') # Vector tile detail at max zoom full_detail = self.get_input_string_port('full_detail', default='') # Vector tile detail at lower zoom levels low_detail = self.get_input_string_port('low_detail', default='') # Get input filename; if there are multiple files, pick one arbitrarily input_dir = self.get_input_data_port('input') filename = [ os.path.join(dp, f) for dp, dn, fn in os.walk(input_dir) for f in fn if ('tif' in f or 'geojson' in f or 'json' in f) ][0] if filename is None: print 'Invalid filename!' return 0 # convert to mbtiles format if geojson if 'geojson' in filename: # if there are filtering criteria present, then filter features print 'Filtering' if criteria: criteria = criteria.split(',') with open(filename) as f: data = json.load(f) filtered_features = [] for feature in data['features']: keep = True # go through each criterion for criterion in criteria: prop, key, value = criterion.split(' ') # retrieve property value this_value = feature['properties'][prop] # some hacking here to handle string- vs. numeric-valued properties if isinstance(this_value, basestring): condition = "'" + this_value + "'" + key + value else: condition = str(this_value) + key + value keep *= eval(condition) if keep: filtered_features.append(feature) with open(filename, 'w') as f: json.dump(FeatureCollection(filtered_features), f) print 'Converting to mbtiles' prefix = filename.split('.geojson')[0] convert = 'tippecanoe -o {}.mbtiles -Z{} -z{}'.format( prefix, min_zoom, max_zoom) if excluded_properties: convert += ' ' + ' '.join( len(excluded_properties) * ['-x {}']).format(*excluded_properties) if full_detail: convert += ' -d{}'.format(full_detail) if low_detail: convert += ' -D{}'.format(low_detail) convert += ' -l {} {}'.format(tileset_name, filename) proc = subprocess.Popen([convert], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate() filename = prefix + '.mbtiles' # Create mapbox uploader object service = mapbox.Uploader() print 'Send upload request' with open(filename, 'r') as src: upload_response = service.upload(src, tileset=tileset_name, name=tileset_name) if upload_response.status_code == 409: for i in range(5): sleep(5) with open(filename, 'r') as src: upload_response = service.upload(src, mapid) if upload_response.status_code != 409: break if upload_response.status_code != 201: print 'Mapbox has not received upload request!' return 0 # Poll the upload API to determine if upload has finished # using upload identifier from 201 response print 'Uploading' upload_id = upload_response.json()['id'] for i in range(timeout / 5): status_response = service.status(upload_id).json() if status_response['complete']: break sleep(5) if tileset_name not in status_response['tileset']: print 'Upload unsuccessful!' return 0 # Now delete completed upload status from upload listing print 'Delete upload listing' delete_response = service.delete(upload_id) for i in range(5): if delete_response.status_code == 204: break else: sleep(5) delete_response = service.delete(upload_id)
def upload(ctx, tileset, datasource, name, patch): """Upload data to Mapbox accounts. Uploaded data lands at https://www.mapbox.com/data/ and can be used in new or existing projects. All endpoints require authentication. You can specify the tileset id and input file $ mapbox upload username.data mydata.geojson Or specify just the tileset id and take an input file on stdin $ cat mydata.geojson | mapbox upload username.data The --name option defines the title as it appears in Studio and defaults to the last part of the tileset id, e.g. "data" Note that the tileset must start with your username. An access token with upload scope is required, see `mapbox --help`. Your account must be flagged in order to use the patch mode feature. """ access_token = (ctx.obj and ctx.obj.get('access_token')) or None service = mapbox.Uploader(access_token=access_token) if name is None: name = tileset.split(".")[-1] if datasource.startswith('https://'): # Skip staging. Note this this only works for specific buckets. res = service.create(datasource, tileset, name=name, patch=patch) else: sourcefile = click.File('rb')(datasource) if hasattr(sourcefile, 'name'): filelen = (1 if sourcefile.name == '<stdin>' else os.stat( sourcefile.name).st_size) else: filelen = (len(sourcefile.getbuffer()) if hasattr( sourcefile, 'getbuffer') else 1) with click.progressbar(length=filelen, label='Uploading data source', fill_char="#", empty_char='-', file=sys.stderr) as bar: def callback(num_bytes): """Update the progress bar""" bar.update(num_bytes) res = service.upload(sourcefile, tileset, name, patch=patch, callback=callback) if res.status_code == 201: click.echo(res.text) else: raise MapboxCLIException(res.text.strip())
def test_resolve_username(): """Username is resolved and deprecation warning raised""" serv = mapbox.Uploader() with pytest.warns(DeprecationWarning): assert serv._resolve_username('foo', 'bar') == 'bar'
account = config['Accountdata'] upload = config['Uploaddata'] username = account['username'] access_token = account['accesstoken'] path = upload['path'] UAV = upload['UAVname'] datatype = upload['datatype'] start = int(upload['startingtileset']) while True: if os.path.isfile(path + '/' + UAV + str(start) + '.' + datatype) == True: print('Data ' + UAV + str(start) + ' found! Preparing to upload:\n') res = mapbox.Uploader(access_token=access_token)._get_credentials() with open(path + '/' + UAV + str(start) + '.' + datatype, 'rb') as src: stage_url = mapbox.Uploader(access_token=access_token).stage(src) print('Data ' + UAV + str(start) + ' successfully staged. Start upload:\n') def print_cb(num_bytes): print("{0} bytes uploaded".format(num_bytes)) with open(path + '/' + UAV + str(start) + '.' + datatype, 'rb') as src: res = mapbox.Uploader(access_token=access_token).upload( src, UAV + str(start), callback=print_cb) print('Data ' + UAV + str(start) + ' successfully uploaded. Search for next:\n') start += 1