class UploadsByConfiguration(ClusteredByConfiguration):
__table_name__ = 'uploads_by_configuration_01'
suite = columns.Text(partition_key=True, required=True)
branch = columns.Text(partition_key=True, required=True)
uuid = columns.BigInt(primary_key=True,
required=True,
clustering_order='DESC')
sdk = columns.Text(primary_key=True, required=True)
commits = columns.Blob(required=True)
test_results = columns.Blob(required=True)
time_uploaded = columns.DateTime(required=True)
upload_version = columns.Integer(required=True)
def unpack(self):
return dict(
commits=[
Commit.from_json(element) for element in json.loads(
UploadContext.from_zip(bytearray(self.commits)))
],
sdk=None if self.sdk == '?' else self.sdk,
test_results=json.loads(
UploadContext.from_zip(bytearray(self.test_results))),
timestamp=calendar.timegm(self.time_uploaded.timetuple()),
version=self.upload_version,
)
class fingerdata(Model):
createtime = columns.DateTime()
modifytime = columns.DateTime()
userid = columns.Text(primary_key=True)
fid = columns.Text()
fingerdata = columns.Blob()
fingerdata_length = columns.Integer()
class MessageData(Model):
node_id = columns.Text(partition_key=True)
date = columns.Date(partition_key=True)
topic = columns.Text(primary_key=True)
created_at = columns.DateTime(primary_key=True)
received_at = columns.DateTime(required=True)
body = columns.Blob(required=True)
class SensorData(Model):
node_id = columns.Text(partition_key=True)
date = columns.Date(partition_key=True)
created_at = columns.DateTime(primary_key=True)
received_at = columns.DateTime()
plugin_id = columns.Text(primary_key=True)
topic = columns.Text()
data = columns.Blob()
class MessageLog(Model):
__options__ = {
'default_time_to_live': 259200,
}
node_id = columns.Text(partition_key=True)
topic = columns.Text(primary_key=True)
created_at = columns.DateTime(primary_key=True)
received_at = columns.DateTime(required=True)
body = columns.Blob(required=True)
class SensorDataLog(Model):
__options__ = {
'default_time_to_live': 60,
}
node_id = columns.Text(partition_key=True)
created_at = columns.DateTime(primary_key=True)
received_at = columns.DateTime()
plugin_id = columns.Text(primary_key=True)
topic = columns.Text()
data = columns.Blob()
class TestResultsBase(ClusteredByConfiguration):
suite = columns.Text(partition_key=True, required=True)
branch = columns.Text(partition_key=True, required=True)
test = columns.Text(partition_key=True, required=True)
expected = columns.Blob(required=True)
actual = columns.Blob(required=True)
details = columns.Text(required=False)
def unpack(self):
result = dict(
uuid=self.uuid,
start_time=calendar.timegm(self.start_time.timetuple()),
expected=Expectations.state_ids_to_string(self.expected),
actual=Expectations.state_ids_to_string(self.actual),
)
for key, value in (json.loads(self.details)
if self.details else {}).items():
if key in result:
continue
result[key] = value
return result
class AllDatatypes(UserType):
a = columns.Ascii()
b = columns.BigInt()
c = columns.Blob()
d = columns.Boolean()
e = columns.DateTime()
f = columns.Decimal()
g = columns.Double()
h = columns.Float()
i = columns.Inet()
j = columns.Integer()
k = columns.Text()
l = columns.TimeUUID()
m = columns.UUID()
n = columns.VarInt()
class ClassifierNetwork(Model):
"""Cassandra Model that represents classifier neural network.
Variables:
------------------
id: UUID.
configuration_id: Id of host aggregate.
model: Neural network model in bytes.
x_maxima: Maximal values of network input data."""
configuration_id = columns.Text(primary_key=True)
model = columns.Blob()
x_maxima = columns.Map(columns.Integer, columns.Double)
class AllDatatypesModel(Model):
id = columns.Integer(primary_key=True)
a = columns.Ascii()
b = columns.BigInt()
c = columns.Blob()
d = columns.Boolean()
e = columns.DateTime()
f = columns.Decimal()
g = columns.Double()
h = columns.Float()
i = columns.Inet()
j = columns.Integer()
k = columns.Text()
l = columns.TimeUUID()
m = columns.UUID()
n = columns.VarInt()
class ArchivesByCommit(ClusteredByConfiguration):
__table_name__ = 'archives_by_commit'
suite = columns.Text(partition_key=True, required=True)
branch = columns.Text(partition_key=True, required=True)
uuid = columns.BigInt(primary_key=True,
required=True,
clustering_order='DESC')
sdk = columns.Text(primary_key=True, required=True)
start_time = columns.BigInt(primary_key=True, required=True)
archive = columns.Blob(required=True)
def unpack(self):
return dict(
uuid=self.uuid,
start_time=self.start_time,
archive=io.BytesIO(self.archive),
)
class Notification(BaseModel):
"""
Table to store notifications queues in cassandra
user_id: user's id to which notification belongs to.
notif_id: time UUID V1 (en.wikipedia.org/wiki/Universally_unique_identifier)
emitter : backend entity that's emitting the message.
type: a single word to describe notification's type: event, info, feedback..
reference: (optional) a reference number previously sent by frontend.
body: could be a simple word or a more complex structure like a json.
"""
user_id = columns.UUID(primary_key=True)
notif_id = columns.TimeUUID(primary_key=True)
emitter = columns.Text()
type = columns.Ascii()
reference = columns.Text()
body = columns.Blob()
class PredictorNetwork(Model):
"""Cassandra Model that represents predictor neural network.
Variables:
------------------
id: UUID.
configuration_id: Id of host aggregate.
image: Name of workload image.
category: Name of workload category.
model: Neural network model in bytes.
x_maxima: Maximal values of network input data.
y_maxima: Maximal values of network output data."""
image = columns.Text(primary_key=True)
category = columns.Text(primary_key=True)
model = columns.Blob()
x_maxima = columns.Map(columns.Integer(), columns.Double())
y_maxima = columns.Map(columns.Integer(), columns.Double())
class NexusTileData(Model):
__table_name__ = 'sea_surface_temp'
tile_id = columns.UUID(primary_key=True)
tile_blob = columns.Blob()
__nexus_tile = None
def _get_nexus_tile(self):
if self.__nexus_tile is None:
self.__nexus_tile = nexusproto.TileData.FromString(self.tile_blob)
return self.__nexus_tile
def get_raw_data_array(self):
nexus_tile = self._get_nexus_tile()
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
return from_shaped_array(the_tile_data.variable_data)
def get_lat_lon_time_data_meta(self):
if self._get_nexus_tile().HasField('grid_tile'):
grid_tile = self._get_nexus_tile().grid_tile
grid_tile_data = np.ma.masked_invalid(from_shaped_array(grid_tile.variable_data))
latitude_data = np.ma.masked_invalid(from_shaped_array(grid_tile.latitude))
longitude_data = np.ma.masked_invalid(from_shaped_array(grid_tile.longitude))
if len(grid_tile_data.shape) == 2:
grid_tile_data = grid_tile_data[np.newaxis, :]
# Extract the meta data
meta_data = {}
for meta_data_obj in grid_tile.meta_data:
name = meta_data_obj.name
meta_array = np.ma.masked_invalid(from_shaped_array(meta_data_obj.meta_data))
if len(meta_array.shape) == 2:
meta_array = meta_array[np.newaxis, :]
meta_data[name] = meta_array
return latitude_data, longitude_data, np.array([grid_tile.time]), grid_tile_data, meta_data
elif self._get_nexus_tile().HasField('swath_tile'):
swath_tile = self._get_nexus_tile().swath_tile
latitude_data = np.ma.masked_invalid(from_shaped_array(swath_tile.latitude)).reshape(-1)
longitude_data = np.ma.masked_invalid(from_shaped_array(swath_tile.longitude)).reshape(-1)
time_data = np.ma.masked_invalid(from_shaped_array(swath_tile.time)).reshape(-1)
# Simplify the tile if the time dimension is the same value repeated
if np.all(time_data == np.min(time_data)):
time_data = np.array([np.min(time_data)])
swath_tile_data = np.ma.masked_invalid(from_shaped_array(swath_tile.variable_data))
tile_data = self._to_standard_index(swath_tile_data,
(len(time_data), len(latitude_data), len(longitude_data)))
# Extract the meta data
meta_data = {}
for meta_data_obj in swath_tile.meta_data:
name = meta_data_obj.name
actual_meta_array = np.ma.masked_invalid(from_shaped_array(meta_data_obj.meta_data))
reshaped_meta_array = self._to_standard_index(actual_meta_array, tile_data.shape)
meta_data[name] = reshaped_meta_array
return latitude_data, longitude_data, time_data, tile_data, meta_data
elif self._get_nexus_tile().HasField('time_series_tile'):
time_series_tile = self._get_nexus_tile().time_series_tile
time_series_tile_data = np.ma.masked_invalid(from_shaped_array(time_series_tile.variable_data))
time_data = np.ma.masked_invalid(from_shaped_array(time_series_tile.time)).reshape(-1)
latitude_data = np.ma.masked_invalid(from_shaped_array(time_series_tile.latitude))
longitude_data = np.ma.masked_invalid(from_shaped_array(time_series_tile.longitude))
reshaped_array = np.ma.masked_all((len(time_data), len(latitude_data), len(longitude_data)))
idx = np.arange(len(latitude_data))
reshaped_array[:, idx, idx] = time_series_tile_data
tile_data = reshaped_array
# Extract the meta data
meta_data = {}
for meta_data_obj in time_series_tile.meta_data:
name = meta_data_obj.name
meta_array = np.ma.masked_invalid(from_shaped_array(meta_data_obj.meta_data))
reshaped_meta_array = np.ma.masked_all((len(time_data), len(latitude_data), len(longitude_data)))
idx = np.arange(len(latitude_data))
reshaped_meta_array[:, idx, idx] = meta_array
meta_data[name] = reshaped_meta_array
return latitude_data, longitude_data, time_data, tile_data, meta_data
else:
raise NotImplementedError("Only supports grid_tile, swath_tile, and time_series_tile")
@staticmethod
def _to_standard_index(data_array, desired_shape):
if desired_shape[0] == 1:
reshaped_array = np.ma.masked_all((desired_shape[1], desired_shape[2]))
row, col = np.indices(data_array.shape)
reshaped_array[np.diag_indices(desired_shape[1], len(reshaped_array.shape))] = data_array[
row.flat, col.flat]
reshaped_array.mask[np.diag_indices(desired_shape[1], len(reshaped_array.shape))] = data_array.mask[
row.flat, col.flat]
reshaped_array = reshaped_array[np.newaxis, :]
else:
reshaped_array = np.ma.masked_all(desired_shape)
row, col = np.indices(data_array.shape)
reshaped_array[np.diag_indices(desired_shape[1], len(reshaped_array.shape))] = data_array[
row.flat, col.flat]
reshaped_array.mask[np.diag_indices(desired_shape[1], len(reshaped_array.shape))] = data_array.mask[
row.flat, col.flat]
return reshaped_array
class LinearModel(Model):
name = columns.Text(primary_key=True, required=True)
model = columns.Blob(required=True)
from cassandra.cqlengine import columns from django_cassandra_engine.models import DjangoCassandraModel # Create your models here. filename = columns.Text(primary_key=True) content = columns.Blob()
class ArchiveChunks(Model):
__table_name__ = 'archive_chunks'
digest = columns.Text(partition_key=True, required=True)
index = columns.Integer(primary_key=True, required=True)
chunk = columns.Blob(required=True)
class NexusTileData(Model):
__table_name__ = 'sea_surface_temp'
tile_id = columns.UUID(primary_key=True)
tile_blob = columns.Blob()
__nexus_tile = None
def _get_nexus_tile(self):
if self.__nexus_tile is None:
self.__nexus_tile = nexusproto.TileData.FromString(self.tile_blob)
return self.__nexus_tile
def get_raw_data_array(self):
nexus_tile = self._get_nexus_tile()
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
return from_shaped_array(the_tile_data.variable_data)
def get_lat_lon_time_data_meta(self):
"""
Retrieve data from data store and metadata from metadata store
for this tile. For gridded tiles, the tile shape of the data
will match the input shape. For example, if the input was a
30x30 tile, all variables will also be 30x30. However, if the
tile is a swath tile, the data will be transformed along the
diagonal of the data matrix. For example, a 30x30 tile would
become 900x900 where the 900 points are along the diagonal.
Multi-variable tile will also include an extra dimension in the
data array. For example, a 30 x 30 x 30 array would be
transformed to N x 30 x 30 x 30 where N is the number of
variables in this tile.
latitude_data, longitude_data, np.array([grid_tile.time]), grid_tile_data, meta_data, is_multi_var
:return: latitude data
:return: longitude data
:return: time data
:return: data
:return: meta data dictionary
:return: boolean flag, True if this tile has more than one variable
"""
is_multi_var = False
if self._get_nexus_tile().HasField('grid_tile'):
grid_tile = self._get_nexus_tile().grid_tile
grid_tile_data = np.ma.masked_invalid(
from_shaped_array(grid_tile.variable_data))
latitude_data = np.ma.masked_invalid(
from_shaped_array(grid_tile.latitude))
longitude_data = np.ma.masked_invalid(
from_shaped_array(grid_tile.longitude))
if len(grid_tile_data.shape) == 2:
grid_tile_data = grid_tile_data[np.newaxis, :]
# Extract the meta data
meta_data = {}
for meta_data_obj in grid_tile.meta_data:
name = meta_data_obj.name
meta_array = np.ma.masked_invalid(
from_shaped_array(meta_data_obj.meta_data))
if len(meta_array.shape) == 2:
meta_array = meta_array[np.newaxis, :]
meta_data[name] = meta_array
return latitude_data, longitude_data, np.array(
[grid_tile.time]), grid_tile_data, meta_data, is_multi_var
elif self._get_nexus_tile().HasField('swath_tile'):
swath_tile = self._get_nexus_tile().swath_tile
latitude_data = np.ma.masked_invalid(
from_shaped_array(swath_tile.latitude)).reshape(-1)
longitude_data = np.ma.masked_invalid(
from_shaped_array(swath_tile.longitude)).reshape(-1)
time_data = np.ma.masked_invalid(from_shaped_array(
swath_tile.time)).reshape(-1)
# Simplify the tile if the time dimension is the same value repeated
if np.all(time_data == np.min(time_data)):
time_data = np.array([np.min(time_data)])
swath_tile_data = np.ma.masked_invalid(
from_shaped_array(swath_tile.variable_data))
tile_data = self._to_standard_index(
swath_tile_data,
(len(time_data), len(latitude_data), len(longitude_data)))
# Extract the meta data
meta_data = {}
for meta_data_obj in swath_tile.meta_data:
name = meta_data_obj.name
actual_meta_array = np.ma.masked_invalid(
from_shaped_array(meta_data_obj.meta_data))
reshaped_meta_array = self._to_standard_index(
actual_meta_array, tile_data.shape)
meta_data[name] = reshaped_meta_array
return latitude_data, longitude_data, time_data, tile_data, meta_data, is_multi_var
elif self._get_nexus_tile().HasField('time_series_tile'):
time_series_tile = self._get_nexus_tile().time_series_tile
time_series_tile_data = np.ma.masked_invalid(
from_shaped_array(time_series_tile.variable_data))
time_data = np.ma.masked_invalid(
from_shaped_array(time_series_tile.time)).reshape(-1)
latitude_data = np.ma.masked_invalid(
from_shaped_array(time_series_tile.latitude))
longitude_data = np.ma.masked_invalid(
from_shaped_array(time_series_tile.longitude))
reshaped_array = np.ma.masked_all(
(len(time_data), len(latitude_data), len(longitude_data)))
idx = np.arange(len(latitude_data))
reshaped_array[:, idx, idx] = time_series_tile_data
tile_data = reshaped_array
# Extract the meta data
meta_data = {}
for meta_data_obj in time_series_tile.meta_data:
name = meta_data_obj.name
meta_array = np.ma.masked_invalid(
from_shaped_array(meta_data_obj.meta_data))
reshaped_meta_array = np.ma.masked_all(
(len(time_data), len(latitude_data), len(longitude_data)))
idx = np.arange(len(latitude_data))
reshaped_meta_array[:, idx, idx] = meta_array
meta_data[name] = reshaped_meta_array
return latitude_data, longitude_data, time_data, tile_data, meta_data, is_multi_var
elif self._get_nexus_tile().HasField('swath_multi_variable_tile'):
swath_tile = self._get_nexus_tile().swath_multi_variable_tile
is_multi_var = True
latitude_data = np.ma.masked_invalid(
from_shaped_array(swath_tile.latitude)).reshape(-1)
longitude_data = np.ma.masked_invalid(
from_shaped_array(swath_tile.longitude)).reshape(-1)
time_data = np.ma.masked_invalid(from_shaped_array(
swath_tile.time)).reshape(-1)
# Simplify the tile if the time dimension is the same value repeated
if np.all(time_data == np.min(time_data)):
time_data = np.array([np.min(time_data)])
swath_tile_data = np.ma.masked_invalid(
from_shaped_array(swath_tile.variable_data))
desired_shape = (
len(time_data),
len(latitude_data),
len(longitude_data),
)
tile_data = self._to_standard_index(swath_tile_data,
desired_shape,
is_multi_var=True)
# Extract the meta data
meta_data = {}
for meta_data_obj in swath_tile.meta_data:
name = meta_data_obj.name
actual_meta_array = np.ma.masked_invalid(
from_shaped_array(meta_data_obj.meta_data))
reshaped_meta_array = self._to_standard_index(
actual_meta_array, tile_data.shape)
meta_data[name] = reshaped_meta_array
return latitude_data, longitude_data, time_data, tile_data, meta_data, is_multi_var
elif self._get_nexus_tile().HasField('grid_multi_variable_tile'):
grid_multi_variable_tile = self._get_nexus_tile(
).grid_multi_variable_tile
is_multi_var = True
grid_tile_data = np.ma.masked_invalid(
from_shaped_array(grid_multi_variable_tile.variable_data))
latitude_data = np.ma.masked_invalid(
from_shaped_array(grid_multi_variable_tile.latitude))
longitude_data = np.ma.masked_invalid(
from_shaped_array(grid_multi_variable_tile.longitude))
# If there are 3 dimensions, that means the time dimension
# was squeezed. Add back in
if len(grid_tile_data.shape) == 3:
grid_tile_data = np.expand_dims(grid_tile_data, axis=1)
# If there are 4 dimensions, that means the time dimension
# is present. Move the multivar dimension.
if len(grid_tile_data.shape) == 4:
grid_tile_data = np.moveaxis(grid_tile_data, -1, 0)
# Extract the meta data
meta_data = {}
for meta_data_obj in grid_multi_variable_tile.meta_data:
name = meta_data_obj.name
meta_array = np.ma.masked_invalid(
from_shaped_array(meta_data_obj.meta_data))
if len(meta_array.shape) == 2:
meta_array = meta_array[np.newaxis, :]
meta_data[name] = meta_array
return latitude_data, longitude_data, np.array([
grid_multi_variable_tile.time
]), grid_tile_data, meta_data, is_multi_var
else:
raise NotImplementedError(
"Only supports grid_tile, swath_tile, swath_multi_variable_tile, and time_series_tile"
)
@staticmethod
def _to_standard_index(data_array, desired_shape, is_multi_var=False):
"""
Transform swath data to a standard format where data runs along
diagonal of ND matrix and the non-diagonal data points are
masked
:param data_array: The data array to be transformed
:param desired_shape: The desired shape of the resulting array
:param is_multi_var: True if this is a multi-variable tile
:type data_array: np.array
:type desired_shape: tuple
:type is_multi_var: bool
:return: Reshaped array
:rtype: np.array
"""
if desired_shape[0] == 1:
reshaped_array = np.ma.masked_all(
(desired_shape[1], desired_shape[2]))
row, col = np.indices(data_array.shape)
reshaped_array[np.diag_indices(
desired_shape[1],
len(reshaped_array.shape))] = data_array[row.flat, col.flat]
reshaped_array.mask[np.diag_indices(
desired_shape[1],
len(reshaped_array.shape))] = data_array.mask[row.flat,
col.flat]
reshaped_array = reshaped_array[np.newaxis, :]
elif is_multi_var == True:
# Break the array up by variable. Translate shape from
# len(times) x len(latitudes) x len(longitudes) x num_vars,
# to
# num_vars x len(times) x len(latitudes) x len(longitudes)
reshaped_data_array = np.moveaxis(data_array, -1, 0)
reshaped_array = []
for variable_data_array in reshaped_data_array:
variable_reshaped_array = np.ma.masked_all(desired_shape)
row, col = np.indices(variable_data_array.shape)
variable_reshaped_array[np.diag_indices(
desired_shape[1],
len(variable_reshaped_array.shape))] = variable_data_array[
row.flat, col.flat]
variable_reshaped_array.mask[np.diag_indices(
desired_shape[1], len(variable_reshaped_array.shape)
)] = variable_data_array.mask[row.flat, col.flat]
reshaped_array.append(variable_reshaped_array)
else:
reshaped_array = np.ma.masked_all(desired_shape)
row, col = np.indices(data_array.shape)
reshaped_array[np.diag_indices(
desired_shape[1],
len(reshaped_array.shape))] = data_array[row.flat, col.flat]
reshaped_array.mask[np.diag_indices(
desired_shape[1],
len(reshaped_array.shape))] = data_array.mask[row.flat,
col.flat]
return reshaped_array
class Record(Model):
key = columns.Bytes(primary_key=True)
enc_values = columns.List(columns.Blob())
class ObjectKeyValue(Model):
key = columns.UUID(primary_key=True)
value = columns.Blob()