def _purge_cache(self, user, collection):
if self.cache is None:
return
cache_key = _key(user, collection, "meta")
try:
self.cache.delete(cache_key)
except CacheError:
logger.error("Unable to delete a cache entry")
def _purge_cache(self, user, collection):
if self.cache is None:
return
cache_key = _key(user, collection, 'meta')
try:
self.cache.delete(cache_key)
except CacheError:
logger.error('Unable to delete a cache entry')
def _set_cached_metadata(self, session, user, collection, data, etag):
key = collection + "::meta"
doc = session.set(key, json.dumps(data), if_match=etag)
if self.cache is not None:
cache_key = _key(user, collection, "meta")
try:
self.cache.set(cache_key, doc, self.cache_ttl)
except CacheError:
logger.error("Unable to write the metadata in the cache.")
def _set_cached_metadata(self, session, user, collection, data, etag):
key = collection + "::meta"
doc = session.set(key, json.dumps(data), if_match=etag)
if self.cache is not None:
cache_key = _key(user, collection, 'meta')
try:
self.cache.set(cache_key, doc, self.cache_ttl)
except CacheError:
logger.error('Unable to write the metadata in the cache.')
def add_applications(self, user, collection, applications, token):
self._check_token(token)
res = self._execute(queries.IS_DEL, user=user, collection=collection)
deleted = res.fetchone()
res.close()
if deleted is not None:
self._execute(queries.REMOVE_DEL, user=user, collection=collection)
now = int(round_time() * 100)
# let's see if we have an uuid
res = self._execute(queries.GET_UUID, user=user, collection=collection)
res = res.fetchone()
if res is None:
# we need to create one
uuid = '%s-%s' % (now, collection)
self._execute(queries.ADD_UUID,
user=user,
collection=collection,
uuid=uuid)
else:
uuid = res.uuid
# the *real* storage will do bulk inserts of course
for app in applications:
origin = app['origin']
res = self._execute(queries.GET_BY_ORIGIN_QUERY,
user=user,
collection=collection,
origin=origin)
res = res.fetchone()
if res is None:
self._execute(queries.PUT_QUERY,
user=user,
collection=collection,
last_modified=now,
data=json.dumps(app),
origin=app['origin'])
else:
## FIXME: for debugging
if res.data == json.dumps(app):
## This is a logic error on the client:
logger.error(('Bad attempt to update an application '
' to overwrite itself: %r') % app['origin'])
self._execute(queries.UPDATE_BY_ORIGIN_QUERY,
user=user,
collection=collection,
id=res.id,
data=json.dumps(app),
last_modified=now)
def add_applications(self, user, collection, applications, token):
self._check_token(token)
res = self._execute(queries.IS_DEL, user=user, collection=collection)
deleted = res.fetchone()
res.close()
if deleted is not None:
self._execute(queries.REMOVE_DEL, user=user, collection=collection)
now = int(round_time() * 100)
# let's see if we have an uuid
res = self._execute(queries.GET_UUID, user=user,
collection=collection)
res = res.fetchone()
if res is None:
# we need to create one
uuid = '%s-%s' % (now, collection)
self._execute(queries.ADD_UUID, user=user,
collection=collection, uuid=uuid)
else:
uuid = res.uuid
# the *real* storage will do bulk inserts of course
for app in applications:
origin = app['origin']
res = self._execute(queries.GET_BY_ORIGIN_QUERY, user=user,
collection=collection, origin=origin)
res = res.fetchone()
if res is None:
self._execute(queries.PUT_QUERY, user=user,
collection=collection,
last_modified=now, data=json.dumps(app),
origin=app['origin'])
else:
## FIXME: for debugging
if res.data == json.dumps(app):
## This is a logic error on the client:
logger.error(('Bad attempt to update an application '
' to overwrite itself: %r') % app['origin'])
self._execute(queries.UPDATE_BY_ORIGIN_QUERY, user=user,
collection=collection,
id=res.id, data=json.dumps(app),
last_modified=now)
def _get_cached_metadata(self, session, user, collection):
# getting the cached value if possible
if self.cache is not None:
cache_key = _key(user, collection, "meta")
try:
cached = self.cache.get(cache_key)
except CacheError:
logger.error("Unable to read the metadata in the cache.")
cached = None
if cached is not None:
return cached
# getting the meta document.
doc = session.getitem(collection + "::meta")
# set the cache
if self.cache is not None:
try:
self.cache.set(cache_key, doc, time=self.cache_ttl)
except CacheError:
logger.error("Was unable to cache the metadata.")
return doc
def _get_cached_metadata(self, session, user, collection):
# getting the cached value if possible
if self.cache is not None:
cache_key = _key(user, collection, 'meta')
try:
cached = self.cache.get(cache_key)
except CacheError:
logger.error('Unable to read the metadata in the cache.')
cached = None
if cached is not None:
return cached
# getting the meta document.
doc = session.getitem(collection + "::meta")
# set the cache
if self.cache is not None:
try:
self.cache.set(cache_key, doc, time=self.cache_ttl)
except CacheError:
logger.error('Was unable to cache the metadata.')
return doc
def check_auth(request):
"""Controls the Authorization header and returns the username and the
collection.
Raises a 401 in these cases:
- If the header is not present or unrecognized
- If the request path is not *owned* by that user
- the database token
The header is of the form:
AppSync b64(assertion):b64(username):b64(token)
"""
user = request.matchdict['user']
collection = request.matchdict['collection']
auth = request.environ.get('HTTP_AUTHORIZATION')
mock_browserid = request.registry.get('mock_browserid')
if mock_browserid:
return user, collection, None
if auth is None:
raise HTTPUnauthorized('No authorization provided')
if not auth.startswith('AppSync '):
logger.error('Attempted auth with bad type (not AppSync): %r' % auth)
raise HTTPUnauthorized('Invalid token; expected Authorization type '
'AppSync')
auth = auth[len('AppSync '):].strip()
auth_part = auth.split(':')
if len(auth_part) != 3:
logger.error('Attempted auth with bad value (not x:y:z): %r' % auth)
raise HTTPUnauthorized('Invalid token; invalid format')
try:
auth_part = [b64dec(part) for part in auth_part]
except (binascii.Error, ValueError), e:
logger.error('Attempted auth with invalid base64 content'
': %r (%s)' % (auth, e))
raise HTTPUnauthorized('Invalid token: invalid base64 encoding')
auth_part = auth.split(':')
if len(auth_part) != 3:
logger.error('Attempted auth with bad value (not x:y:z): %r' % auth)
raise HTTPUnauthorized('Invalid token; invalid format')
try:
auth_part = [b64dec(part) for part in auth_part]
except (binascii.Error, ValueError), e:
logger.error('Attempted auth with invalid base64 content'
': %r (%s)' % (auth, e))
raise HTTPUnauthorized('Invalid token: invalid base64 encoding')
assertion, username, dbtoken = auth_part
# let's reject the call if the url is not owned by the user
if user != username:
logger.error('Attempted auth for user=%r for collection '
'user=%r' % (username, user))
raise HTTPUnauthorized('Invalid user')
# need to verify the user signature here
# XXX
return user, collection, dbtoken
def create_auth(assertion, username, token):
auth = 'AppSync %s:%s:%s' % (b64enc(assertion), b64enc(username),
b64enc(token))
return auth
def get_data(request):
"""After authenticating with the server and getting back the URL of the
collection, request::
GET /collections/{user}/{collection}?since=timestamp
`since` is optional; on first sync is should be empty or left off. The
server will return an object::
{until: timestamp,
uuid: someuniquevalue, # using a timestamp XXXX
incomplete: bool, applications: {origin: {...},
...} }
The response may not be complete if there are too many applications.
If this is the case then `incomplete` will be true (it may be left out
if
the response is complete). Another request using `since={until}` will
get further applications (this may need to be repeated many times).
The client should save the value of `until` and use it for subsequent
requests.
In the case of no new items the response will be only::
{until: timestamp}
The client should always start with this GET request and only then send
its own updates. It should ensure that its local timestamp is
sensible in comparison to the value of `until`.
Applications returned may be older than the local applications, in that
case then the client should ignore the server's application and use
its local copy, causing an overwrite. The same is true for deleted
applications; if the local installed copy has a `last_modified` date
newer than the deleted server instance then the server instance
should be ignored (the user reinstalled an application).
**NOTE:** there are some conflicts that may occur, specifically
receipts should be merged.
When an application is added from the server the client should
*locally* set `app.sync` to true (this is in the [application
representation]
(https://developer.mozilla.org/en/OpenWebApps/The_JavaScript_API
#Application_Representation), not the manifest).
You must always retain `last_modified` as you received it from
the server (unless you ignore the application in favor of a
newer local version).
"""
user, collection, dbtoken = check_auth(request)
try:
since = request.GET.get('since', '0')
since = round_time(since)
except TypeError:
raise bad_request(INVALID_SINCE_VALUE)
except ValueError:
logger.error('Bad since %r' % since)
raise bad_request(INVALID_SINCE_VALUE,
'Invalid value for since: %r' % since)
if since.is_nan():
raise bad_request(INVALID_SINCE_VALUE, 'Got NaN value for since')
storage = get_storage(request)
res = {'since': since, 'uuid': storage.get_uuid(user, collection, dbtoken)}
until = 0
apps = []
try:
for index, (last_modified, app) in enumerate(
storage.get_applications(user,
collection,
since,
token=dbtoken)):
if last_modified > until:
until = last_modified
apps.append(app)
res['applications'] = apps
if not until:
until = since
res['until'] = until
except CollectionDeletedError, e:
return {
'collection_deleted': {
'reason': e.reason,
'client_id': e.client_id
}
}
except CollectionDeletedError, e:
return {
'collection_deleted': {
'reason': e.reason,
'client_id': e.client_id
}
}
# do we want to add a X-Sync-Poll ?
cache = get_cache(request)
if cache is not None:
try:
poll_interval = cache.get('X-Sync-Poll')
except CacheError, e:
# a well, nevermind then
logger.error(str(e))
else:
if poll_interval is not None:
request.response.headers['X-Sync-Poll'] = str(poll_interval)
return res
@data.post()
def post_data(request):
"""The client should keep track of the last time it sent updates to the
server, and send updates when there are newer applications.
**NOTE:** there is a case when an update might be lost because of an
update from another device; this would be okay except that the client
doesn't know it needs to re-send that update. How do we confirm that ?
def execute_retry(engine, *args, **kwargs):
try:
return engine.execute(*args, **kwargs)
except (OperationalError, TimeoutError), exc:
retry = '2013' in str(exc)
try:
if retry:
return engine.execute(*args, **kwargs)
else:
# re-raise
raise exc
except (OperationalError, TimeoutError), exc:
err = traceback.format_exc()
logger.error(err)
raise BackendError(str(exc))
class SQLDatabase(object):
implements(IAppSyncDatabase)
def __init__(self, **options):
verifier = options.pop("verifier", None)
if verifier is None:
verifier = vep.RemoteVerifier()
else:
verifier = maybe_resolve_name(verifier)
if callable(verifier):
verifier = verifier()
self._verifier = verifier
def get_data(request):
"""After authenticating with the server and getting back the URL of the
collection, request::
GET /collections/{user}/{collection}?since=timestamp
`since` is optional; on first sync is should be empty or left off. The
server will return an object::
{until: timestamp,
uuid: someuniquevalue, # using a timestamp XXXX
incomplete: bool, applications: {origin: {...},
...} }
The response may not be complete if there are too many applications.
If this is the case then `incomplete` will be true (it may be left out
if
the response is complete). Another request using `since={until}` will
get further applications (this may need to be repeated many times).
The client should save the value of `until` and use it for subsequent
requests.
In the case of no new items the response will be only::
{until: timestamp}
The client should always start with this GET request and only then send
its own updates. It should ensure that its local timestamp is
sensible in comparison to the value of `until`.
Applications returned may be older than the local applications, in that
case then the client should ignore the server's application and use
its local copy, causing an overwrite. The same is true for deleted
applications; if the local installed copy has a `last_modified` date
newer than the deleted server instance then the server instance
should be ignored (the user reinstalled an application).
**NOTE:** there are some conflicts that may occur, specifically
receipts should be merged.
When an application is added from the server the client should
*locally* set `app.sync` to true (this is in the [application
representation]
(https://developer.mozilla.org/en/OpenWebApps/The_JavaScript_API
#Application_Representation), not the manifest).
You must always retain `last_modified` as you received it from
the server (unless you ignore the application in favor of a
newer local version).
"""
user, collection, dbtoken = check_auth(request)
try:
since = request.GET.get('since', '0')
since = round_time(since)
except TypeError:
raise bad_request(INVALID_SINCE_VALUE)
except ValueError:
logger.error('Bad since %r' % since)
raise bad_request(INVALID_SINCE_VALUE,
'Invalid value for since: %r' % since)
if since.is_nan():
raise bad_request(INVALID_SINCE_VALUE,
'Got NaN value for since')
storage = get_storage(request)
res = {'since': since,
'uuid': storage.get_uuid(user, collection, dbtoken)}
until = 0
apps = []
try:
for index, (last_modified, app) in enumerate(
storage.get_applications(user, collection, since,
token=dbtoken)):
if last_modified > until:
until = last_modified
apps.append(app)
res['applications'] = apps
if not until:
until = since
res['until'] = until
except CollectionDeletedError, e:
return {'collection_deleted': {'reason': e.reason,
'client_id': e.client_id}}
if not until:
until = since
res['until'] = until
except CollectionDeletedError, e:
return {'collection_deleted': {'reason': e.reason,
'client_id': e.client_id}}
# do we want to add a X-Sync-Poll ?
cache = get_cache(request)
if cache is not None:
try:
poll_interval = cache.get('X-Sync-Poll')
except CacheError, e:
# a well, nevermind then
logger.error(str(e))
else:
if poll_interval is not None:
request.response.headers['X-Sync-Poll'] = str(poll_interval)
return res
@data.post()
def post_data(request):
"""The client should keep track of the last time it sent updates to the
server, and send updates when there are newer applications.
**NOTE:** there is a case when an update might be lost because of an
update from another device; this would be okay except that the client
doesn't know it needs to re-send that update. How do we confirm that ?
def execute_retry(engine, *args, **kwargs):
try:
return engine.execute(*args, **kwargs)
except (OperationalError, TimeoutError), exc:
retry = '2013' in str(exc)
try:
if retry:
return engine.execute(*args, **kwargs)
else:
# re-raise
raise exc
except (OperationalError, TimeoutError), exc:
err = traceback.format_exc()
logger.error(err)
raise BackendError(str(exc))
class SQLDatabase(object):
implements(IAppSyncDatabase)
def __init__(self, **options):
verifier = options.pop("verifier", None)
if verifier is None:
verifier = vep.RemoteVerifier()
else:
verifier = maybe_resolve_name(verifier)
if callable(verifier):
verifier = verifier()
self._verifier = verifier
