def delete_multi(self, uss_id, z, grids):
"""Sets multiple GridCells metadata by removing the entry for the USS.
Reads data from zookeeper, including a snapshot token. The
snapshot token is used as a reference when writing to ensure
the data has not been updated between read and write.
Args:
uss_id: is the plain text identifier for the USS
z: zoom level in slippy tile format
grids: list of (x,y) tiles to delete
Returns:
JSend formatted response (https://labs.omniti.com/labs/jsend)
"""
log.debug('Deleting multiple grid metadata for %s...', uss_id)
try:
if not uss_id:
raise ValueError('Invalid uss_id for deleting multi')
for x, y in grids:
if slippy_util.validate_slippy(z, x, y):
(content, metadata) = self._get_raw(z, x, y)
if metadata:
m = uss_metadata.USSMetadata(content)
m.remove_operator(uss_id)
# TODO(pelletierb): Automatically retry on delete
status = self._set_raw(z, x, y, m, metadata.version)
else:
raise ValueError('Invalid slippy grids for lookup')
result = self.get_multi(z, grids)
except ValueError as e:
result = self._format_status_code_to_jsend(400, e.message)
return result
def _get_multi_raw(self, z, grids):
"""Gets the raw content and metadata for multiple GridCells from zookeeper.
Args:
z: zoom level in slippy tile format
grids: list of (x,y) tiles to retrieve
Returns:
content: Combined USS metadata
syncs: list of sync tokens in the same order as the grids
Raises:
IndexError: if it cannot find anything in zookeeper
ValueError: if the grid data is not in the right format
"""
log.debug('Getting multiple grid metadata for %s...', str(grids))
combined_meta = None
syncs = []
for x, y in grids:
if slippy_util.validate_slippy(z, x, y):
(content, metadata) = self._get_raw(z, x, y)
if metadata:
combined_meta += uss_metadata.USSMetadata(content)
syncs.append(metadata.last_modified_transaction_id)
else:
raise IndexError('Unable to find metadata in platform')
else:
raise ValueError('Invalid slippy grids for lookup')
if len(syncs) == 0:
raise IndexError('Unable to find metadata in platform')
return combined_meta, syncs
def delete_multi(self, z, grids, uss_id):
"""Sets multiple GridCells metadata by removing the entry for the USS.
Removes the operator from multiple cells. Does not return 404 on
not finding the USS in a cell, since this should be a remove all
type function, as some cells might have the ussid and some might not.
Args:
z: zoom level in slippy tile format
grids: list of (x,y) tiles to delete
uss_id: is the plain text identifier for the USS
Returns:
JSend formatted response (https://labs.omniti.com/labs/jsend)
"""
log.debug('Deleting multiple grid metadata for %s...', uss_id)
try:
if not uss_id:
raise ValueError('Invalid uss_id for deleting multi')
for x, y in grids:
if slippy_util.validate_slippy(z, x, y):
(content, metadata) = self._get_raw(z, x, y)
if metadata:
m = uss_metadata.USSMetadata(content)
m.remove_operator(uss_id)
# TODO(pelletierb): Automatically retry on delete
status = self._set_raw(z, x, y, m, metadata.version)
else:
raise ValueError('Invalid slippy grids for lookup')
result = self.get_multi(z, grids)
except ValueError as e:
result = self._format_status_code_to_jsend(400, e.message)
return result
def set(self, z, x, y, sync_token, uss_id, ws_scope, operation_format,
operation_ws, earliest_operation, latest_operation):
"""Sets the metadata for a GridCell.
Writes data, using the snapshot token for confirming data
has not been updated since it was last read.
Args:
z: zoom level in slippy tile format
x: x tile number in slippy tile format
y: y tile number in slippy tile format
sync_token: token retrieved in the original GET GridCellMetadata,
uss_id: plain text identifier for the USS,
ws_scope: scope to use to obtain OAuth token,
operation_format: output format for operation ws (i.e. NASA, GUTMA),
operation_ws: submitting USS endpoint where all flights in
this cell can be retrieved from,
earliest_operation: lower bound of active or planned flight timestamp,
used for quick filtering conflicts.
latest_operation: upper bound of active or planned flight timestamp,
used for quick filtering conflicts.
Returns:
JSend formatted response (https://labs.omniti.com/labs/jsend)
"""
status = 500
if self._validate_slippy(z, x, y):
# first we have to get the cell
status = 0
(content, metadata) = self._get_raw(z, x, y)
if metadata:
# Quick check of the token, another is done on the actual set to be sure
# but this check fails early and fast
if str(metadata.last_modified_transaction_id) == str(
sync_token):
try:
m = uss_metadata.USSMetadata(content)
log.debug('Setting metadata for %s...', uss_id)
if not m.upsert_operator(
uss_id, ws_scope, operation_format,
operation_ws, earliest_operation,
latest_operation):
log.error(
'Failed setting operator for %s with token %s...',
uss_id, str(sync_token))
raise ValueError
status = self._set_raw(z, x, y, m, uss_id, sync_token)
except ValueError:
status = 424
else:
status = 409
else:
status = 404
else:
status = 400
if status == 200:
# Success, now get the metadata back to send back
result = self.get(z, x, y)
else:
result = self._format_status_code_to_jsend(status)
return result
def testUSSmetadatAddition(self):
a = uss_metadata.USSMetadata()
a.upsert_operator('uss-a', 'scope-a', 'NASA', 'http://a.com/uss',
'2018-01-01', '2018-01-02', 10, 1, 1)
b1 = uss_metadata.USSMetadata()
b1.upsert_operator('uss-b', 'scope-b', 'NASA', 'http://b.com/uss',
'2018-01-01', '2018-01-02', 10, 1, 1)
b2 = uss_metadata.USSMetadata()
b2.upsert_operator('uss-b', 'scope-b', 'NASA', 'http://b.com/uss',
'2018-01-01', '2018-01-02', 10, 1, 2)
usss = a + b1 + b2
self.assertEqual(3, len(usss.operators))
with self.assertRaises(ValueError):
usss = a + a
ax = uss_metadata.USSMetadata()
ax.upsert_operator('uss-a', 'scope-ax', 'NASA', 'http://ax.com/uss',
'2018-01-03', '2018-01-04', 10, 1, 1)
with self.assertRaises(ValueError):
usss = a + ax
def delete(self, z, x, y, uss_id):
"""Sets the metadata for a GridCell by removing the entry for the USS.
Args:
z: zoom level in slippy tile format
x: x tile number in slippy tile format
y: y tile number in slippy tile format
uss_id: is the plain text identifier for the USS
Returns:
JSend formatted response (https://labs.omniti.com/labs/jsend)
"""
status = 500
if self._validate_slippy(z, x, y):
# first we have to get the cell
(content, metadata) = self._get_raw(z, x, y)
if metadata:
try:
m = uss_metadata.USSMetadata(content)
m.remove_operator(uss_id)
# TODO(pelletierb): Automatically retry on delete
status = self._set_raw(
z, x, y, m, uss_id,
metadata.last_modified_transaction_id)
except ValueError:
status = 424
else:
status = 404
else:
status = 400
if status == 200:
# Success, now get the metadata back to send back
(content, metadata) = self._get_raw(z, x, y)
result = {
'status': 'success',
'sync_token': metadata.last_modified_transaction_id,
'data': m.to_json()
}
else:
result = self._format_status_code_to_jsend(status)
return result
def get(self, z, x, y):
"""Gets the metadata and snapshot token for a GridCell.
Reads data from zookeeper, including a snapshot token. The
snapshot token is used as a reference when writing to ensure
the data has not been updated between read and write.
Args:
z: zoom level in slippy tile format
x: x tile number in slippy tile format
y: y tile number in slippy tile format
Returns:
JSend formatted response (https://labs.omniti.com/labs/jsend)
"""
# TODO(hikevin): Change to use our own error codes and let the server
# convert them to http error codes. For now, this is
# at least in a standard JSend format.
status = 500
if slippy_util.validate_slippy(z, x, y):
(content, metadata) = self._get_raw(z, x, y)
if metadata:
try:
m = uss_metadata.USSMetadata(content)
status = 200
result = {
'status': 'success',
'sync_token': metadata.last_modified_transaction_id,
'data': m.to_json()
}
except ValueError:
status = 424
else:
status = 404
else:
status = 400
if status != 200:
result = self._format_status_code_to_jsend(status)
return result
def _set_multi_raw(self, z, grids, sync_tokens, uss_id, ws_scope,
operation_format, operation_ws, earliest_operation,
latest_operation):
"""Grabs the lock and updates the raw content for multiple GridCells
Args:
z: zoom level in slippy tile format
grids: list of (x,y) tiles to retrieve
sync_tokens: list of the sync tokens received during get operation
uss_id: plain text identifier for the USS,
ws_scope: scope to use to obtain OAuth token,
operation_format: output format for operation ws (i.e. NASA, GUTMA),
operation_ws: submitting USS endpoint where all flights in
this cell can be retrieved from,
earliest_operation: lower bound of active or planned flight timestamp,
used for quick filtering conflicts.
latest_operation: upper bound of active or planned flight timestamp,
used for quick filtering conflicts.
Raises:
IndexError: if it cannot find anything in zookeeper
ValueError: if the grid data is not in the right format
"""
log.debug('Setting multiple grid metadata for %s...', str(grids))
try:
contents = []
for i in range(len(grids)):
# First, get and update them all in memory, validate the sync_token
x = grids[i][0]
y = grids[i][1]
sync_token = sync_tokens[i]
path = '%s/%s/%s/%s/%s' % (GRID_PATH, str(z), str(x), str(y),
USS_METADATA_FILE)
(content, metadata) = self._get_raw(z, x, y)
if str(metadata.last_modified_transaction_id) == str(
sync_token):
log.debug('Sync_token matches for %d, %d...', x, y)
m = uss_metadata.USSMetadata(content)
if not m.upsert_operator(
uss_id, ws_scope, operation_format, operation_ws,
earliest_operation, latest_operation, z, x, y):
raise ValueError('Failed to set operator content')
contents.append((path, m, metadata.version))
else:
log.error(
'Sync token from USS (%s) does not match token from zk (%s)...',
str(sync_token),
str(metadata.last_modified_transaction_id))
raise KeyError('Composite sync_token has changed')
# Now, start a transaction to update them all
# the version will catch any changes and roll back any attempted
# updates to the grids
log.debug('Starting transaction to write all grids at once...')
t = self.zk.transaction()
for path, m, version in contents:
t.set_data(path, json.dumps(m.to_json()), version)
log.debug('Committing transaction...')
results = t.commit()
if isinstance(results[0], RolledBackError):
raise KeyError(
'Rolled back multi-grid transaction due to grid change')
log.debug('Committed transaction successfully.')
except (KeyError, ValueError, IndexError) as e:
log.error('Error caught in set_multi_raw %s.', e.message)
raise e