Beispiel #1
0
def test_delete():
    job = json.loads(upload_response_body)
    responses.add(
        responses.DELETE,
        'https://api.mapbox.com/uploads/v1/{0}/{1}?access_token={2}'.format(username, job['id'], access_token),
        match_querystring=True,
        body=None, status=204,
        content_type='application/json')

    res = mapbox.Uploader(access_token=access_token).delete(job)
    assert res.status_code == 204

    res = mapbox.Uploader(access_token=access_token).delete(job['id'])
    assert res.status_code == 204
Beispiel #2
0
def test_status():
    job = json.loads(upload_response_body)
    responses.add(
        responses.GET,
        'https://api.mapbox.com/uploads/v1/{0}/{1}?access_token={2}'.format(username, job['id'], access_token),
        match_querystring=True,
        body=upload_response_body, status=200,
        content_type='application/json')

    res = mapbox.Uploader(access_token=access_token).status(job)
    assert res.status_code == 200
    res = mapbox.Uploader(access_token=access_token).status(job['id'])
    assert res.status_code == 200
    status = res.json()
    assert job == status
def test_upload_tileset_validation_specialchar():
    with pytest.raises(mapbox.errors.ValidationError):
        with open('tests/moors.json', 'rb') as src:
            # limited to a-z0-9-_ chars
            mapbox.Uploader(access_token=access_token).upload(src,
                                                              'username.&#!',
                                                              name='test1')
def test_create_name():
    upload_response_body = """
        {"progress": 0,
        "modified": "date.test",
        "error": null,
        "tileset": "testuser.test1",
        "complete": false,
        "owner": "testuser",
        "created": "date.test",
        "id": "id.test",
        "name": "testname"}"""

    def request_callback(request):
        payload = json.loads(request.body.decode())
        assert payload['name'] == "testname"
        return (201, {}, upload_response_body)

    responses.add_callback(
        responses.POST,
        'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format(
            username, access_token),
        match_querystring=True,
        callback=request_callback)

    res = mapbox.Uploader(access_token=access_token).create(
        'http://example.com/test.json', 'testuser.test1', name="testname")
    assert res.status_code == 201
    job = res.json()
    assert job['name'] == "testname"
def test_stage(monkeypatch):

    monkeypatch.setattr(mapbox.services.uploads, 'boto3_session', MockSession)

    # Credentials
    query_body = """
       {{"key": "_pending/{username}/key.test",
         "accessKeyId": "ak.test",
         "bucket": "tilestream-tilesets-production",
         "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test",
         "secretAccessKey": "sak.test",
         "sessionToken": "st.test"}}""".format(username=username)
    responses.add(
        responses.POST,
        'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token={1}'.
        format(username, access_token),
        match_querystring=True,
        body=query_body,
        status=200,
        content_type='application/json')

    with open('tests/moors.json', 'rb') as src:
        stage_url = mapbox.Uploader(access_token=access_token).stage(src)
    assert stage_url.startswith(
        "https://tilestream-tilesets-production.s3.amazonaws.com/_pending")
def test_big_stage(tmpdir, monkeypatch):
    """Files larger than 1M are multipart uploaded."""

    monkeypatch.setattr(mapbox.services.uploads, 'boto3_session', MockSession)

    # Credentials
    query_body = """
       {{"key": "_pending/{username}/key.test",
         "accessKeyId": "ak.test",
         "bucket": "tilestream-tilesets-production",
         "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test",
         "secretAccessKey": "sak.test",
         "sessionToken": "st.test"}}""".format(username=username)

    responses.add(
        responses.POST,
        'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token={1}'.
        format(username, access_token),
        match_querystring=True,
        body=query_body,
        status=200,
        content_type='application/json')

    # Make a temp file larger than 1MB.
    bigfile = tmpdir.join('big.txt')
    bigfile.write(','.join(('num' for num in range(1000000))))
    assert bigfile.size() > 1000000

    with bigfile.open(mode='rb') as src:
        stage_url = mapbox.Uploader(access_token=access_token).stage(src)
    assert stage_url.startswith(
        "https://tilestream-tilesets-production.s3.amazonaws.com/_pending")
Beispiel #7
0
def test_upload_tileset_validation():
    with pytest.raises(mapbox.errors.ValidationError):
        with open('tests/moors.json', 'rb') as src:
            mapbox.Uploader(access_token=access_token).upload(src,
                                                              'a' * 65,
                                                              name='test1',
                                                              patch=True)
def test_get_credentials():
    query_body = """
       {{"key": "_pending/{username}/key.test",
         "accessKeyId": "ak.test",
         "bucket": "tilestream-tilesets-production",
         "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test",
         "secretAccessKey": "sak.test",
         "sessionToken": "st.test"}}""".format(username=username)

    responses.add(
        responses.POST,
        'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token={1}'.
        format(username, access_token),
        match_querystring=True,
        body=query_body,
        status=200,
        content_type='application/json')

    res = mapbox.Uploader(access_token=access_token)._get_credentials()

    assert res.status_code == 200
    creds = res.json()
    assert username in creds['url']
    for k in [
            'key', 'bucket', 'url', 'accessKeyId', 'secretAccessKey',
            'sessionToken'
    ]:
        assert k in creds.keys()
Beispiel #9
0
def create_tileset(ctx, dataset, tileset, name):
    """Create a vector tileset from a dataset.

        $ mapbox datasets create-tileset dataset-id username.data

    Note that the tileset must start with your username and the dataset
    must be one that you own. To view processing status, visit
    https://www.mapbox.com/data/. You may not generate another tilesets
    from the same dataset until the first processing job has completed.

    All endpoints require authentication. An access token with
    `uploads:write` scope is required, see `mapbox --help`.
    """

    access_token = (ctx.obj and ctx.obj.get('access_token')) or None
    service = mapbox.Uploader(access_token=access_token)

    uri = "mapbox://datasets/{username}/{dataset}".format(
        username=tileset.split('.')[0], dataset=dataset)

    res = service.create(uri, tileset, name)

    if res.status_code == 201:
        click.echo(res.text)
    else:
        raise MapboxCLIException(res.text.strip())
Beispiel #10
0
def test_upload_error(monkeypatch):
    """Upload a file and create a tileset, fails with 409"""

    monkeypatch.setattr(mapbox.services.uploads, 'boto3_session', MockSession)

    # Credentials
    query_body = """
       {{"key": "_pending/{username}/key.test",
         "accessKeyId": "ak.test",
         "bucket": "tilestream-tilesets-production",
         "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test",
         "secretAccessKey": "sak.test",
         "sessionToken": "st.test"}}""".format(username=username)

    responses.add(
        responses.GET,
        'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token={1}'.format(username, access_token),
        match_querystring=True,
        body=query_body, status=200,
        content_type='application/json')

    responses.add(
        responses.POST,
        'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format(username, access_token),
        match_querystring=True,
        body="", status=409,
        content_type='application/json')

    with open('tests/moors.json', 'r') as src:
        res = mapbox.Uploader(access_token=access_token).upload(src, 'test1')

    assert res.status_code == 409
Beispiel #11
0
def test_upload_tileset_validation():
    with pytest.raises(mapbox.errors.ValidationError):
        with open('tests/moors.json', 'rb') as src:
            # limited to 32 chars, try 40
            mapbox.Uploader(access_token=access_token).upload(src,
                                                              'username.' +
                                                              'aA' * 20,
                                                              name='test1')
Beispiel #12
0
def upload(ctx, args, name):
    """Upload data to Mapbox accounts.
    All endpoints require authentication.
    Uploaded data lands at https://www.mapbox.com/data/
    and can be used in new or existing projects.

    You can specify the input file and tileset id

      $ mapbox upload mydata.geojson username.data

    Or specify just the tileset id and take an input file on stdin

      $ cat mydata.geojson | mapbox upload username.data

    The --name option defines the title as it appears in Studio
    and defaults to the last part of the tileset id, e.g. "data"

    Note that the tileset must start with your username.
    An access token with upload scope is required, see `mapbox --help`.
    """
    access_token = (ctx.obj and ctx.obj.get('access_token')) or None

    service = mapbox.Uploader(access_token=access_token)

    if len(args) == 1:
        # Tileset specified, file from stdin
        click.echo("Reading data from stdin (Hit Ctl-C to cancel) ...",
                   err=True)
        infile = BytesIO(click.File("rb")("-").read())
        tileset = args[0]
    elif len(args) == 2:
        # Infile and Tileset are specified
        try:
            infile = click.File("rb")(args[0])
        except click.ClickException:
            raise click.UsageError(
                "Could not open file: {0} "
                "(check order of command arguments: INFILE TILESET)".format(
                    args[0]))

        tileset = args[1]
    else:
        raise click.UsageError(
            "Must provide either one argument (TILESET) or two (INFILE TILESET)"
        )

    if name is None:
        name = tileset.split(".")[-1]

    try:
        res = service.upload(infile, tileset, name)
    except (mapbox.errors.ValidationError, IOError) as exc:
        raise click.BadParameter(str(exc))

    if res.status_code == 201:
        click.echo(res.text)
    else:
        raise MapboxCLIException(res.text.strip())
Beispiel #13
0
def test_upload_tileset_validation_username():
    # even with 60 chars, the addition of the
    # testuser username puts it over 64 chars
    with pytest.raises(mapbox.errors.ValidationError):
        with open('tests/moors.json', 'rb') as src:
            mapbox.Uploader(access_token=access_token).upload(src,
                                                              'a' * 60,
                                                              name='test1',
                                                              patch=True)
Beispiel #14
0
def test_create():
    responses.add(
        responses.POST,
        'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format(username, access_token),
        match_querystring=True,
        body=upload_response_body, status=201,
        content_type='application/json')

    res = mapbox.Uploader(access_token=access_token).create(
        'http://example.com/test.json', 'test1')  # without username prefix
    assert res.status_code == 201
    job = res.json()
    assert job['tileset'] == "{0}.test1".format(username)

    res2 = mapbox.Uploader(access_token=access_token).create(
        'http://example.com/test.json', 'testuser.test1')  # also takes full tileset
    assert res2.status_code == 201
    job = res2.json()
    assert job['tileset'] == "{0}.test1".format(username)
Beispiel #15
0
def upload(filename, upload_id):
    uploader = mapbox.Uploader()

    def attempt():
        return attempt_upload(uploader, filename, upload_id)

    response = retry(attempt, retries=5, pause=5, succeeded=succeeded)

    if response.status_code != 201:
        raise MapboxError(response)

    return response
Beispiel #16
0
def test_list():
    responses.add(
        responses.GET,
        'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format(username, access_token),
        match_querystring=True,
        body="[{0}]".format(upload_response_body), status=200,
        content_type='application/json')

    res = mapbox.Uploader(access_token=access_token).list()
    assert res.status_code == 200
    uploads = res.json()
    assert len(uploads) == 1
    assert json.loads(upload_response_body) in uploads
Beispiel #17
0
def test_upload_patch(monkeypatch):
    """Upload a file and create a tileset in patch mode"""

    monkeypatch.setattr(mapbox.services.uploads, 'boto3_session', MockSession)

    def ensure_patch(request):
        payload = json.loads(request.body.decode())
        assert payload['patch']
        headers = {}
        return (201, headers, upload_response_body)

    # Credentials
    query_body = """
       {{"key": "_pending/{username}/key.test",
         "accessKeyId": "ak.test",
         "bucket": "tilestream-tilesets-production",
         "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test",
         "secretAccessKey": "sak.test",
         "sessionToken": "st.test"}}""".format(username=username)

    responses.add(
        responses.POST,
        'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token={1}'.
        format(username, access_token),
        match_querystring=True,
        body=query_body,
        status=200,
        content_type='application/json')

    responses.add_callback(
        responses.POST,
        'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format(
            username, access_token),
        callback=ensure_patch,
        match_querystring=True,
        content_type='application/json')

    with open('tests/moors.json', 'rb') as src:
        res = mapbox.Uploader(access_token=access_token).upload(
            src, 'testuser.test1', name='test1', patch=True)

    assert res.status_code == 201
    job = res.json()
    assert job['tileset'] == "{0}.test1".format(username)
Beispiel #18
0
def test_upload(monkeypatch):
    """Upload a file and create a tileset"""

    monkeypatch.setattr(mapbox.services.uploads, 'boto3_session', MockSession)

    # Credentials
    query_body = """
       {{"key": "_pending/{username}/key.test",
         "accessKeyId": "ak.test",
         "bucket": "tilestream-tilesets-production",
         "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test",
         "secretAccessKey": "sak.test",
         "sessionToken": "st.test"}}""".format(username=username)

    responses.add(
        responses.POST,
        'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token={1}'.
        format(username, access_token),
        match_querystring=True,
        body=query_body,
        status=200,
        content_type='application/json')

    responses.add(
        responses.POST,
        'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format(
            username, access_token),
        match_querystring=True,
        body=upload_response_body,
        status=201,
        content_type='application/json')

    def print_cb(num_bytes):
        print("{0} bytes uploaded".format(num_bytes))

    with open('tests/moors.json', 'rb') as src:
        res = mapbox.Uploader(access_token=access_token).upload(
            src, 'test1', callback=print_cb)

    assert res.status_code == 201
    job = res.json()
    assert job['tileset'] == "{0}.test1".format(username)
Beispiel #19
0
    def run(self):
        wx.PostEvent(self._notify_window, ResultEvent('---STARTED---\n'))
        self.log.write(self.st + '---STARTED---\n')
        wx.PostEvent(self._notify_window,
                     ResultEvent('Path:  ' + self.pfad + '\n'))
        self.log.write(self.st + 'Path:  ' + self.pfad + '\n')
        wx.PostEvent(self._notify_window,
                     ResultEvent('Dronename:  ' + self.dronename + '\n'))
        self.log.write(self.st + 'Dronename:  ' + self.dronename + '\n')
        wx.PostEvent(self._notify_window,
                     ResultEvent('Datatype:  ' + self.datatype + '\n'))
        self.log.write(self.st + 'Datatype:  ' + self.datatype + '\n')
        wx.PostEvent(
            self._notify_window,
            ResultEvent('Starting Tileset:  ' + str(self.starttileset) + '\n'))
        self.log.write(self.st + 'Starting Tileset:  ' +
                       str(self.starttileset) + '\n')
        #j = 1

        while True:
            if os.path.isfile(self.pfad + '/' + self.dronename +
                              str(self.starttileset) + self.datatype) == True:
                wx.PostEvent(
                    self._notify_window,
                    ResultEvent('Data ' + self.dronename +
                                str(self.starttileset) +
                                ' found! Preparing to upload:\n'))
                self.log.write(self.st + 'Data ' + self.dronename +
                               str(self.starttileset) +
                               ' found! Preparing to upload:\n')

                res = mapbox.Uploader(
                    access_token=self.access_token)._get_credentials()
                with open(
                        self.pfad + '/' + self.dronename +
                        str(self.starttileset) + self.datatype, 'rb') as src:
                    stage_url = mapbox.Uploader(
                        access_token=self.access_token).stage(src)

                wx.PostEvent(
                    self._notify_window,
                    ResultEvent('Data ' + self.dronename +
                                str(self.starttileset) +
                                ' successfully staged. Start upload:\n'))
                self.log.write(self.st + 'Data ' + self.dronename +
                               str(self.starttileset) +
                               ' successfully staged. Start upload:\n')

                def print_cb(num_bytes):
                    wx.PostEvent(
                        self._notify_window,
                        ResultEvent('{0} bytes uploaded\n'.format(num_bytes)))
                    self.log.write(self.st +
                                   '{0} bytes uploaded\n'.format(num_bytes))

                with open(
                        self.pfad + '/' + self.dronename +
                        str(self.starttileset) + self.datatype, 'rb') as src:
                    res = mapbox.Uploader(
                        access_token=self.access_token).upload(
                            src,
                            self.dronename + str(self.starttileset),
                            callback=print_cb)
                    wx.PostEvent(
                        self._notify_window,
                        ResultEvent(
                            'Data ' + self.dronename + str(self.starttileset) +
                            ' successfully uploaded. Search for next:\n'))
                    self.log.write(
                        self.st + 'Data ' + self.dronename +
                        str(self.starttileset) +
                        ' successfully uploaded. Search for next:\n')
                    self.starttileset += 1
                    time.sleep(3)

            else:
                wx.PostEvent(
                    self._notify_window,
                    ResultEvent('Data ' + self.dronename +
                                str(self.starttileset) +
                                ' not found. Sleep 3 seconds:\n'))
                self.log.write(self.st + 'Data ' + self.dronename +
                               str(self.starttileset) +
                               ' not found. Sleep 3 seconds:\n')
                time.sleep(3)
            if self._want_abort:
                wx.PostEvent(self._notify_window,
                             ResultEvent('---STOPPED---\n'))
                self.log.write(self.st + '---STOPPED---\n')
                wx.PostEvent(self._notify_window, ResultEvent(None))
                return
Beispiel #20
0
def test_invalid_fileobj():
    """Must be file object, not path"""
    with pytest.raises(mapbox.errors.InvalidFileError):
        mapbox.Uploader(access_token=access_token).upload(
            'tests/moors.json', 'test1')
Beispiel #21
0
def test_class_attrs():
    """Get expected class attr values"""
    serv = mapbox.Uploader()
    assert serv.api_name == 'uploads'
    assert serv.api_version == 'v1'
Beispiel #22
0
def test_create_tileset_validation():
    # even with 60 chars, the addition of the username puts it over 64 chars
    with pytest.raises(mapbox.errors.ValidationError):
        mapbox.Uploader(access_token=access_token).create(
            'http://example.com/test.json', 'a' * 60)
Beispiel #23
0
geometry = [Point(xy) for xy in zip(ottawa['Longitude'], ottawa['Latitude'])]
ottawa = gp.GeoDataFrame(ottawa, geometry=geometry)
ottawa.crs = {'init': 'epsg:4326'}
ottawa = ottawa.to_crs(gdb.crs)

gdb = gdb.loc[(gdb['is_active'] == 1) & (gdb['aoi_code'] == 'QC_LowerOttawa')]
ottawa_flood = gp.sjoin(ottawa, gdb, how='inner', op='within')
ottawa_flood = ottawa_flood.loc[:, [
    'NodeID', 'StreetNumber', 'StreetName', 'geometry'
]]
ottawa_flood = ottawa_flood.to_crs({'init': 'epsg:4326'})
ottawa_flood.to_file('ottawa_flood.geojson',
                     driver='GeoJSON',
                     encoding="utf-8")

gdb = gp.overlay(gdb, gdb, how='union')
gdb.geometry = gdb.geometry.simplify(10)
gdb = gdb.explode()
gdb = gdb.to_crs({'init': 'epsg:4326'})

gdb.to_file('extent.geojson', driver='GeoJSON', encoding="utf-8")

u = mb.Uploader(access_token=sk)

url = u.stage(open('ottawa_flood.geojson', 'rb'))
job = u.create(url, 'ottawa_flood').json()

url = u.stage(open('extent.geojson', 'rb'))
job = u.create(url, 'extent').json()
Beispiel #24
0
# In[16]:

print(output_file_path_tiles)

# In[20]:

command = "tippecanoe -o {} -Z {} -z {} {}.geojson".format(
    output_file_path_tiles, MIN_ZOOM_LIMIT, MAX_ZOOM_LIMIT, output_file_path)

# In[23]:

result = subprocess.check_output(command, shell=True)

# In[24]:

service = mapbox.Uploader()

# In[25]:

mapid = output_dataset_name

# In[26]:

print(mapid)

# In[27]:

with open(output_file_path_tiles, 'rb') as src:
    upload_resp = service.upload(src, mapid)

# In[28]:
    def invoke(self):

        # Token
        token = self.get_input_string_port('token')
        os.environ['MAPBOX_ACCESS_TOKEN'] = token

        # Tileset name
        tileset_name = self.get_input_string_port('tileset_name')

        # Timeout (in seconds)
        timeout = int(self.get_input_string_port('timeout', default='600'))

        # The following string inputs apply only to vector tiles

        # Min zoom
        min_zoom = self.get_input_string_port('min_zoom', default='0')

        # Max zoom
        max_zoom = self.get_input_string_port('max_zoom', default='14')

        # Properties to be excluded
        excluded_properties = self.get_input_string_port('excluded_properties',
                                                         default='').split(',')

        # Filtering criteria
        criteria = self.get_input_string_port('criteria', default='')

        # Vector tile detail at max zoom
        full_detail = self.get_input_string_port('full_detail', default='')

        # Vector tile detail at lower zoom levels
        low_detail = self.get_input_string_port('low_detail', default='')

        # Get input filename; if there are multiple files, pick one arbitrarily
        input_dir = self.get_input_data_port('input')
        filename = [
            os.path.join(dp, f) for dp, dn, fn in os.walk(input_dir)
            for f in fn if ('tif' in f or 'geojson' in f or 'json' in f)
        ][0]

        if filename is None:
            print 'Invalid filename!'
            return 0

        # convert to mbtiles format if geojson
        if 'geojson' in filename:

            # if there are filtering criteria present, then filter features
            print 'Filtering'
            if criteria:
                criteria = criteria.split(',')
                with open(filename) as f:
                    data = json.load(f)
                    filtered_features = []
                    for feature in data['features']:
                        keep = True
                        # go through each criterion
                        for criterion in criteria:
                            prop, key, value = criterion.split(' ')
                            # retrieve property value
                            this_value = feature['properties'][prop]
                            # some hacking here to handle string- vs. numeric-valued properties
                            if isinstance(this_value, basestring):
                                condition = "'" + this_value + "'" + key + value
                            else:
                                condition = str(this_value) + key + value
                            keep *= eval(condition)
                        if keep:
                            filtered_features.append(feature)
                with open(filename, 'w') as f:
                    json.dump(FeatureCollection(filtered_features), f)

            print 'Converting to mbtiles'
            prefix = filename.split('.geojson')[0]
            convert = 'tippecanoe -o {}.mbtiles -Z{} -z{}'.format(
                prefix, min_zoom, max_zoom)
            if excluded_properties:
                convert += ' ' + ' '.join(
                    len(excluded_properties) *
                    ['-x {}']).format(*excluded_properties)
            if full_detail:
                convert += ' -d{}'.format(full_detail)
            if low_detail:
                convert += ' -D{}'.format(low_detail)

            convert += ' -l {} {}'.format(tileset_name, filename)
            proc = subprocess.Popen([convert],
                                    shell=True,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            out, err = proc.communicate()
            filename = prefix + '.mbtiles'

        # Create mapbox uploader object
        service = mapbox.Uploader()

        print 'Send upload request'
        with open(filename, 'r') as src:
            upload_response = service.upload(src,
                                             tileset=tileset_name,
                                             name=tileset_name)
            if upload_response.status_code == 409:
                for i in range(5):
                    sleep(5)
                    with open(filename, 'r') as src:
                        upload_response = service.upload(src, mapid)
                        if upload_response.status_code != 409:
                            break

        if upload_response.status_code != 201:
            print 'Mapbox has not received upload request!'
            return 0

        # Poll the upload API to determine if upload has finished
        # using upload identifier from 201 response
        print 'Uploading'
        upload_id = upload_response.json()['id']
        for i in range(timeout / 5):
            status_response = service.status(upload_id).json()
            if status_response['complete']:
                break
            sleep(5)

        if tileset_name not in status_response['tileset']:
            print 'Upload unsuccessful!'
            return 0

        # Now delete completed upload status from upload listing
        print 'Delete upload listing'
        delete_response = service.delete(upload_id)
        for i in range(5):
            if delete_response.status_code == 204:
                break
            else:
                sleep(5)
                delete_response = service.delete(upload_id)
Beispiel #26
0
def upload(ctx, tileset, datasource, name, patch):
    """Upload data to Mapbox accounts.

    Uploaded data lands at https://www.mapbox.com/data/ and can be used
    in new or existing projects. All endpoints require authentication.

    You can specify the tileset id and input file

      $ mapbox upload username.data mydata.geojson

    Or specify just the tileset id and take an input file on stdin

      $ cat mydata.geojson | mapbox upload username.data

    The --name option defines the title as it appears in Studio and
    defaults to the last part of the tileset id, e.g. "data"

    Note that the tileset must start with your username. An access
    token with upload scope is required, see `mapbox --help`.

    Your account must be flagged in order to use the patch mode
    feature.
    """
    access_token = (ctx.obj and ctx.obj.get('access_token')) or None

    service = mapbox.Uploader(access_token=access_token)

    if name is None:
        name = tileset.split(".")[-1]

    if datasource.startswith('https://'):
        # Skip staging. Note this this only works for specific buckets.
        res = service.create(datasource, tileset, name=name, patch=patch)

    else:
        sourcefile = click.File('rb')(datasource)

        if hasattr(sourcefile, 'name'):
            filelen = (1 if sourcefile.name == '<stdin>' else os.stat(
                sourcefile.name).st_size)
        else:
            filelen = (len(sourcefile.getbuffer()) if hasattr(
                sourcefile, 'getbuffer') else 1)

        with click.progressbar(length=filelen,
                               label='Uploading data source',
                               fill_char="#",
                               empty_char='-',
                               file=sys.stderr) as bar:

            def callback(num_bytes):
                """Update the progress bar"""
                bar.update(num_bytes)

            res = service.upload(sourcefile,
                                 tileset,
                                 name,
                                 patch=patch,
                                 callback=callback)

    if res.status_code == 201:
        click.echo(res.text)
    else:
        raise MapboxCLIException(res.text.strip())
Beispiel #27
0
def test_resolve_username():
    """Username is resolved and deprecation warning raised"""
    serv = mapbox.Uploader()
    with pytest.warns(DeprecationWarning):
        assert serv._resolve_username('foo', 'bar') == 'bar'
Beispiel #28
0
account = config['Accountdata']
upload = config['Uploaddata']

username = account['username']
access_token = account['accesstoken']

path = upload['path']
UAV = upload['UAVname']
datatype = upload['datatype']
start = int(upload['startingtileset'])

while True:
    if os.path.isfile(path + '/' + UAV + str(start) + '.' + datatype) == True:
        print('Data ' + UAV + str(start) + ' found! Preparing to upload:\n')

        res = mapbox.Uploader(access_token=access_token)._get_credentials()
        with open(path + '/' + UAV + str(start) + '.' + datatype, 'rb') as src:
            stage_url = mapbox.Uploader(access_token=access_token).stage(src)

        print('Data ' + UAV + str(start) +
              ' successfully staged. Start upload:\n')

        def print_cb(num_bytes):
            print("{0} bytes uploaded".format(num_bytes))

        with open(path + '/' + UAV + str(start) + '.' + datatype, 'rb') as src:
            res = mapbox.Uploader(access_token=access_token).upload(
                src, UAV + str(start), callback=print_cb)
            print('Data ' + UAV + str(start) +
                  ' successfully uploaded. Search for next:\n')
            start += 1