class DonationOrganizationsByPrefix(tdb_cassandra.View):
_use_db = True
_value_type = "bytes" # disable tdb_cassandra's deserialization
_write_consistency_level = tdb_cassandra.CL.ALL
_read_consistency_level = tdb_cassandra.CL.ONE
_compare_with = types.CompositeType(
types.FloatType(reversed=True),
types.UTF8Type(),
)
_extra_schema_creation_args = {
"key_validation_class": "UTF8Type",
"default_validation_class": "UTF8Type",
}
@classmethod
def byPrefix(cls, prefix):
stripped = donate_utils.normalize_query(prefix)
if not stripped:
return []
try:
results = cls._cf.get(stripped, column_count=MAX_COLUMNS)
except NotFoundException:
return []
return [
Organization(json.loads(data))
for key, data in results.iteritems()
]
def initializeTableLayout(keyspace, server_list, replicationStrategy,
strategyOptions, localDCName, credentials):
sys_manager = SystemManager(server_list[0], credentials=credentials)
# Make sure the the keyspace exists
if keyspace not in sys_manager.list_keyspaces():
sys_manager.create_keyspace(keyspace, replicationStrategy,
strategyOptions)
cf_defs = sys_manager.get_keyspace_column_families(keyspace)
# Create UTF8 CF's
for tablename in ["global_nodes", "metadata"]:
if tablename not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, tablename)
if localDCName:
dcNodes = "dc_%s_nodes" % (localDCName,)
if dcNodes not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, dcNodes)
else:
# TODO Log we do not have the DC name
pass
if "node_slices" not in cf_defs.keys():
sys_manager.create_column_family(
keyspace,
"node_slices",
super=False,
comparator_type=pycassa_types.LongType(),
key_validation_class=pycassa_types.UTF8Type(),
default_validation_class=pycassa_types.LongType()
)
class Meetup(tdb_cassandra.Thing):
_use_db = True
_timestamp_prop = None
_defaults = {
# one of:
# - "prep": finding meetups and printing badges
# - "active": badges and connections
# - "after": just connections
# - "closed": all done
"state": "prep",
}
_extra_schema_creation_args = dict(
key_validation_class=types.AsciiType(),
default_validation_class=types.UTF8Type(),
column_validation_classes={
"date": types.DateType(),
},
)
_read_consistency_level = tdb_cassandra.CL.ONE
_write_consistency_level = tdb_cassandra.CL.ALL
_date_props = ("date", )
@classmethod
def _new(cls, codename, title, meetup_date):
meetup = cls(
_id=codename,
title=title,
date=meetup_date,
body="",
)
meetup._commit()
return meetup
def createUTF8ColumnFamily(sys_manager, keyspace, tablename, ts_table=False):
"""Create column family with UTF8Type comparator, value and key."""
# Default options for the tree-storage tables
kw_options = {'super': False, 'comparator_type': pycassa_types.UTF8Type(),
'key_validation_class': pycassa_types.UTF8Type(),
'default_validation_class': pycassa_types.UTF8Type()}
# A ts* table needs to have a bunch of different settings since that's where
# the actual data is going to be stored
# - Use DTCS compaction to delete old data (requires 2.0.11+)
# - Set the correct CQL table fields
# - Set gc_grace_seconds to 0 since we never update/delete data once it's written
# - DTCS options:
# - timestamp_resolution: MICROSECONDS due to Pycassa and CQL's default timestamp resolution
# - tombstone_threhold: 0.1 as an attempt to be more aggressive when it comes to compaction
# - base_time_seconds: 60 groups sstables into 60 second blocks
# CASSANDRA-8417 indicates that 60 seconds is a more reasonable default
# - tombstone_compaction_interval: 1 is a Cassandra default. Included in case it needs tuning
if ts_table:
table_options = {
'compaction_strategy': 'DateTieredCompactionStrategy',
'compaction_strategy_options': {
'timestamp_resolution': 'MICROSECONDS',
'max_sstable_age_days': '365',
'tombstone_compaction_interval': '1',
'tombstone_threshold': '0.1',
'base_time_seconds': '60'},
'comparator_type': pycassa_types.LongType(),
'key_validation_class': pycassa_types.UTF8Type(),
'default_validation_class': pycassa_types.FloatType(),
'gc_grace_seconds': '1800'}
kw_options.update(table_options)
sys_manager.create_column_family(
keyspace,
tablename,
**kw_options
)
""" Replication strategy that puts a number of replicas in each datacenter """
OLD_NETWORK_TOPOLOGY_STRATEGY = 'OldNetworkTopologyStrategy'
"""
Original replication strategy for putting a number of replicas in each datacenter.
This was originally called 'RackAwareStrategy'.
"""
KEYS_INDEX = IndexType.KEYS
""" A secondary index type where each indexed value receives its own row """
BYTES_TYPE = types.BytesType()
LONG_TYPE = types.LongType()
INT_TYPE = types.IntegerType()
ASCII_TYPE = types.AsciiType()
UTF8_TYPE = types.UTF8Type()
TIME_UUID_TYPE = types.TimeUUIDType()
LEXICAL_UUID_TYPE = types.LexicalUUIDType()
COUNTER_COLUMN_TYPE = types.CounterColumnType()
DOUBLE_TYPE = types.DoubleType()
FLOAT_TYPE = types.FloatType()
DECIMAL_TYPE = types.DecimalType()
BOOLEAN_TYPE = types.BooleanType()
DATE_TYPE = types.DateType()
class SystemManager(object):
"""
Lets you examine and modify schema definitions as well as get basic
information about the cluster.
This class is mainly designed to be used manually in a python shell,
def syncdb(keyspace=None):
"""
Create Cassandra keyspace, CF, SCF
"""
if not keyspace:
keyspace = FLAGS.get("cassandra_keyspace", "synaps_test")
serverlist = FLAGS.get("cassandra_server_list")
replication_factor = FLAGS.get("cassandra_replication_factor")
manager = pycassa.SystemManager(server=serverlist[0])
strategy_options = {'replication_factor': replication_factor}
# create keyspace
LOG.info(_("cassandra syncdb is started for keyspace(%s)" % keyspace))
if keyspace not in manager.list_keyspaces():
LOG.info(_("cassandra keyspace %s does not exist.") % keyspace)
manager.create_keyspace(keyspace,
strategy_options=strategy_options)
LOG.info(_("cassandra keyspace %s is created.") % keyspace)
else:
property = manager.get_keyspace_properties(keyspace)
# check strategy_option
if not (strategy_options == property.get('strategy_options')):
manager.alter_keyspace(keyspace,
strategy_options=strategy_options)
LOG.info(
_("cassandra keyspace strategy options is updated - %s" %
str(strategy_options)))
# create CF, SCF
column_families = manager.get_keyspace_column_families(keyspace)
if 'Metric' not in column_families.keys():
manager.create_column_family(
keyspace=keyspace,
name='Metric',
comparator_type=pycassa.ASCII_TYPE,
key_validation_class=pycassa.LEXICAL_UUID_TYPE,
column_validation_classes={
'project_id': pycassa.UTF8_TYPE,
'name': pycassa.UTF8_TYPE,
'namespace': pycassa.UTF8_TYPE,
'unit': pycassa.UTF8_TYPE,
'dimensions': pycassa.UTF8_TYPE,
'updated_timestamp': pycassa.DATE_TYPE,
'created_timestamp': pycassa.DATE_TYPE
})
manager.create_index(keyspace=keyspace,
column_family='Metric',
column='project_id',
value_type=types.UTF8Type())
manager.create_index(keyspace=keyspace,
column_family='Metric',
column='name',
value_type=types.UTF8Type())
manager.create_index(keyspace=keyspace,
column_family='Metric',
column='namespace',
value_type=types.UTF8Type())
manager.create_index(keyspace=keyspace,
column_family='Metric',
column='dimensions',
value_type=types.UTF8Type())
manager.create_index(keyspace=keyspace,
column_family='Metric',
column='updated_timestamp',
value_type=types.DateType())
manager.create_index(keyspace=keyspace,
column_family='Metric',
column='created_timestamp',
value_type=types.DateType())
if 'StatArchive' not in column_families.keys():
manager.create_column_family(
keyspace=keyspace,
name='StatArchive',
super=True,
key_validation_class=pycassa.LEXICAL_UUID_TYPE,
comparator_type=pycassa.ASCII_TYPE,
subcomparator_type=pycassa.DATE_TYPE,
default_validation_class=pycassa.DOUBLE_TYPE)
if 'MetricAlarm' not in column_families.keys():
manager.create_column_family(
keyspace=keyspace,
name='MetricAlarm',
key_validation_class=pycassa.LEXICAL_UUID_TYPE,
comparator_type=pycassa.ASCII_TYPE,
column_validation_classes={
'metric_key': pycassa.LEXICAL_UUID_TYPE,
'project_id': pycassa.UTF8_TYPE,
'actions_enabled': pycassa.BOOLEAN_TYPE,
'alarm_actions': pycassa.UTF8_TYPE,
'alarm_arn': pycassa.UTF8_TYPE,
'alarm_configuration_updated_timestamp': pycassa.DATE_TYPE,
'alarm_description': pycassa.UTF8_TYPE,
'alarm_name': pycassa.UTF8_TYPE,
'comparison_operator': pycassa.UTF8_TYPE,
'dimensions': pycassa.UTF8_TYPE,
'evaluation_periods': pycassa.INT_TYPE,
'insufficient_data_actions': pycassa.UTF8_TYPE,
'metric_name': pycassa.UTF8_TYPE,
'namespace': pycassa.UTF8_TYPE,
'ok_actions': pycassa.UTF8_TYPE,
'period': pycassa.INT_TYPE,
'state_reason': pycassa.UTF8_TYPE,
'state_reason_data': pycassa.UTF8_TYPE,
'state_updated_timestamp': pycassa.DATE_TYPE,
'state_value': pycassa.UTF8_TYPE,
'statistic': pycassa.UTF8_TYPE,
'threshold': pycassa.DOUBLE_TYPE,
'unit': pycassa.UTF8_TYPE
})
manager.create_index(keyspace=keyspace,
column_family='MetricAlarm',
column='project_id',
value_type=types.UTF8Type())
manager.create_index(keyspace=keyspace,
column_family='MetricAlarm',
column='metric_key',
value_type=types.LexicalUUIDType())
manager.create_index(keyspace=keyspace,
column_family='MetricAlarm',
column='alarm_name',
value_type=types.UTF8Type())
manager.create_index(keyspace=keyspace,
column_family='MetricAlarm',
column='state_updated_timestamp',
value_type=types.DateType())
manager.create_index(
keyspace=keyspace,
column_family='MetricAlarm',
column='alarm_configuration_updated_timestamp',
value_type=types.DateType())
manager.create_index(keyspace=keyspace,
column_family='MetricAlarm',
column='state_value',
value_type=types.UTF8Type())
manager.create_index(keyspace=keyspace,
column_family='MetricAlarm',
column='period',
value_type=types.IntegerType())
manager.create_index(keyspace=keyspace,
column_family='MetricAlarm',
column='statistic',
value_type=types.UTF8Type())
if 'AlarmHistory' not in column_families.keys():
manager.create_column_family(
keyspace=keyspace,
name='AlarmHistory',
key_validation_class=pycassa.LEXICAL_UUID_TYPE,
comparator_type=pycassa.ASCII_TYPE,
column_validation_classes={
'project_id': pycassa.UTF8_TYPE,
'alarm_key': pycassa.LEXICAL_UUID_TYPE,
'alarm_name': pycassa.UTF8_TYPE,
'history_data': pycassa.UTF8_TYPE,
'history_item_type': pycassa.UTF8_TYPE,
'history_summary': pycassa.UTF8_TYPE,
'timestamp': pycassa.DATE_TYPE,
})
manager.create_index(keyspace=keyspace,
column_family='AlarmHistory',
column='project_id',
value_type=types.UTF8Type())
manager.create_index(keyspace=keyspace,
column_family='AlarmHistory',
column='alarm_key',
value_type=types.LexicalUUIDType())
manager.create_index(keyspace=keyspace,
column_family='AlarmHistory',
column='alarm_name',
value_type=types.UTF8Type())
manager.create_index(keyspace=keyspace,
column_family='AlarmHistory',
column='history_item_type',
value_type=types.UTF8Type())
manager.create_index(keyspace=keyspace,
column_family='AlarmHistory',
column='timestamp',
value_type=types.DateType())
if 'AlarmCounter' not in column_families.keys():
manager.create_column_family(
keyspace=keyspace,
name='AlarmCounter',
default_validation_class=pycassa.COUNTER_COLUMN_TYPE,
key_validation_class=pycassa.UTF8_TYPE)
if 'NotificationGroup' not in column_families.keys():
manager.create_column_family(
keyspace=keyspace,
name='NotificationGroup',
key_validation_class=pycassa.UTF8_TYPE,
comparator_type=pycassa.UTF8_TYPE,
default_validation_class=pycassa.UTF8_TYPE)
LOG.info(_("cassandra syncdb has finished"))