def _insert_into_datastore(self):
"""Transfer datafiles from S3 into the postgres datastore.
:return: the url of the fiscal datapackage on Open-Spending
"""
query = dict(jwt=self.user.token, datapackage=self._descriptor_s3_url)
response = upload_package(params=query)
handle(response)
log.info('Congratuations, %s was uploaded successfully!', self)
log.info('You can find you fiscal datapackage here: %s', self.url)
return self.url
def _insert_into_datastore(self):
"""Transfer datafiles from S3 into the postgres datastore.
:return: the url of the fiscal datapackage on Open-Spending
"""
query = dict(jwt=self.user.token, datapackage=self._descriptor_s3_url)
response = upload_package(params=query)
handle(response)
log.info('Congratuations, %s was uploaded successfully!', self)
log.info('You can find you fiscal datapackage here: %s', self.url)
return self.url
def toggle(self, to_state):
"""Toggle public access to a fiscal datapackage
Change the status of a fiscal data package from public to private or
vice-versa. If something went wrong, whilst changing the status, you
will get a :class:`upload.ToggleError`.
:param to_state: the unique name of the datapackage
:return: the new state of the package, i.e. "public" or "private"
"""
publish = True if to_state == 'public' else False
package_id = self.user.id + ':' + self.name
query = dict(
jwt=self.user.permissions['os.datastore']['token'],
id=package_id,
publish=publish
)
answer = handle(toggle_publish(params=query))
if not answer['success']:
message = 'Unable to toggle datapackage to %s'
raise ToggleError(message, to_state)
log.info('%s is now %s', package_id, to_state)
return to_state
def request_new_token():
localhost = 'http://%s:%s' % settings.LOCALHOST
query = dict(next=localhost, callback_url=oauth_callback.url)
response = authenticate_user(query)
authorization = handle(response)
new_thread = Thread(target=listen_for_token)
prompt_user(authorization)
local_server = new_thread.run()
local_server.join()
log.info('Well done, you have a now token!')
def request_new_token():
localhost = 'http://%s:%s' % settings.LOCALHOST
query = dict(next=localhost, callback_url=oauth_callback.url)
response = authenticate_user(query)
authorization = handle(response)
new_thread = Thread(target=listen_for_token)
prompt_user(authorization)
local_server = new_thread.run()
local_server.join()
log.info('Well done, you have a now token!')
def _request_s3_upload(self):
"""Request AWS S3 upload urls for all files.
"""
response = request_upload(params=dict(jwt=self.user.token), json=self.filedata)
files = handle(response)['filedata']
for path, info in files.items():
message = '%s is ready for upload to %s'
log.info(message, path, info['upload_url'])
query = {k: v[0] for k, v in info['upload_query'].items()}
yield info['upload_url'], path, query, self._get_header(path, info['type'])
def _request_permissions(self):
"""Request permissions for Open-Spending services.
"""
permissions = {}
for service in OS_SERVICES:
query = dict(jwt=self.token, service=service)
response = authorize_user(params=query)
permission = handle(response)
permissions.update({service: permission})
return permissions
def _request_permissions(self):
"""Request permissions for Open-Spending services.
"""
permissions = {}
for service in OS_SERVICES:
query = dict(jwt=self.token, service=service)
response = authorize_user(params=query)
permission = handle(response)
permissions.update({service: permission})
return permissions
def _request_s3_upload(self):
"""Request AWS S3 upload urls for all files.
"""
response = request_upload(params=dict(jwt=self.user.token),
json=self.filedata)
files = handle(response)['filedata']
for path, info in files.items():
message = '%s is ready for upload to %s'
log.info(message, path, info['upload_url'])
query = {k: v[0] for k, v in info['upload_query'].items()}
yield info['upload_url'], path, query, self._get_header(
path, info['type'])
def _request_authentication(self):
"""Ask Open-Spending if the token is valid.
"""
query = dict(jwt=self.token)
response = authenticate_user(params=query)
authentication = handle(response)
if not authentication['authenticated']:
message = 'Token has expired: request a new one'
log.error(message)
raise InvalidToken(message)
name = authentication['profile']['name']
log.info('Hello %s! You are logged into Open-Spending', name)
return authentication
def _request_authentication(self):
"""Ask Open-Spending if the token is valid.
"""
query = dict(jwt=self.token)
response = authenticate_user(params=query)
authentication = handle(response)
if not authentication['authenticated']:
message = 'Token has expired: request a new one'
log.error(message)
raise InvalidToken(message)
name = authentication['profile']['name']
log.info('Hello %s! You are logged into Open-Spending', name)
return authentication
def search(global_query=None, keyed_query=None, private=True, limit=None):
"""Query the packages on Open-Spending.
You can search a package by `title`, `author`, `description`, `regionCode`,
`countryCode` or`cityCode`. You can match all these fields at once with the
magic `q` key.
If authentication-token was provided, then private packages from the
authenticated user will also be included. Otherwise, only public packages
will be returned. You can limit the size of your results with the `size`
parameter.
:param global_query: an expression to search for across all keys
:param keyed_query: a `dict` of key/value search pairs
:param private: search private datapackages or not (default: True)
:param limit: the number of results returned (default: None)
:return: a dictionary with the results
:rtype: :class: `list` of `dict`
"""
assert global_query or keyed_query
if keyed_query:
_check_keys(keyed_query)
raw_query = _prefix_keys(keyed_query)
else:
raw_query = {}
if global_query:
raw_query.update(q=global_query)
query = _quote_values(raw_query)
if limit:
query.update(size=limit)
if private:
query.update(jwt=User().token)
response = search_packages(params=query)
return handle(response)
def search(global_query=None, keyed_query=None, private=True, limit=None):
"""Query the packages on Open-Spending.
You can search a package by `title`, `author`, `description`, `regionCode`,
`countryCode` or`cityCode`. You can match all these fields at once with the
magic `q` key.
If authentication-token was provided, then private packages from the
authenticated user will also be included. Otherwise, only public packages
will be returned. You can limit the size of your results with the `size`
parameter.
:param global_query: an expression to search for across all keys
:param keyed_query: a `dict` of key/value search pairs
:param private: search private datapackages or not (default: True)
:param limit: the number of results returned (default: None)
:return: a dictionary with the results
:rtype: :class: `list` of `dict`
"""
assert global_query or keyed_query
if keyed_query:
_check_keys(keyed_query)
raw_query = _prefix_keys(keyed_query)
else:
raw_query = {}
if global_query:
raw_query.update(q=global_query)
query = _quote_values(raw_query)
if limit:
query.update(size=limit)
if private:
query.update(jwt=User().token)
response = search_packages(params=query)
return handle(response)
def search(value=None, where=None, private=True, limit=None):
"""Query the packages on Open-Spending.
You can search a package by `title`, `author`, `description`, `regionCode`,
`countryCode` or`cityCode`. You can match all these fields at once with the
magic `q` key.
If authentication-token was provided, then private packages from the
authenticated user will also be included. Otherwise, only public packages
will be returned. You can limit the size of your results with the `size`
parameter.
:param value:
:param where: a `dict` of key value pairs
:param private: show private datapackages
:param limit: the number of results returned
:return: a dictionary with the results
:rtype: :class: `list` of `dict`
"""
assert value or where
query = {}
if value:
query = {"q": value}
if where:
query = _validate(where)
if limit:
query.update(size=limit)
quoted_query = _sanitize(query)
if private:
user = User()
quoted_query.update(jwt=user.token)
response = search_packages(params=quoted_query)
return handle(response)
def search(value=None, where=None, private=True, limit=None):
"""Query the packages on Open-Spending.
You can search a package by `title`, `author`, `description`, `regionCode`,
`countryCode` or`cityCode`. You can match all these fields at once with the
magic `q` key.
If authentication-token was provided, then private packages from the
authenticated user will also be included. Otherwise, only public packages
will be returned. You can limit the size of your results with the `size`
parameter.
:param value:
:param where: a `dict` of key value pairs
:param private: show private datapackages
:param limit: the number of results returned
:return: a dictionary with the results
:rtype: :class: `list` of `dict`
"""
assert value or where
query = {}
if value:
query = {'q': value}
if where:
query = _validate(where)
if limit:
query.update(size=limit)
quoted_query = _sanitize(query)
if private:
user = User()
quoted_query.update(jwt=user.token)
response = search_packages(params=quoted_query)
return handle(response)
def _s3_callback(_, response):
handle(response)
log.info('Successful S3 upload: %s', response.url)
def test_handle_raises_error_if_status_is_400(api_call):
responses.add(api_call.method, api_call.url, body=HTTPError())
with raises(HTTPError):
handle(api_call())
def test_handle_returns_json_if_status_is_200(api_call):
responses.add(api_call.method, api_call.url, body='{"foo": "bar"}')
assert handle(api_call()) == {'foo': 'bar'}
def _s3_callback(_, response):
handle(response)
log.info('Successful S3 upload: %s', response.url)