def handle_noargs(self, **options):
db = options.get('database')
engine = settings.DATABASES.get(db, {}).get('ENGINE', '')
# Call regular syncdb if engine is different from ours
if engine != 'django_cassandra_engine':
return super(Command, self).handle_noargs(**options)
if django.VERSION < (1, 7):
self._import_management()
connection = connections[db]
connection.connect()
options = connection.settings_dict.get('OPTIONS', {})
replication_opts = options.get('replication', {})
keyspace = connection.settings_dict['NAME']
self.stdout.write('Creating keyspace %s..' % keyspace)
create_keyspace(keyspace, **replication_opts)
for app_name, app_models \
in connection.introspection.cql_models.iteritems():
for model in app_models:
self.stdout.write('Syncing %s.%s' % (app_name, model.__name__))
sync_table(model, create_missing_keyspace=False)
def create(cassa_keyspace='pyhackers'):
assert cassa_keyspace != '' or cassa_keyspace is not None
logging.warn("Creating/Sync'ing {}".format(cassa_keyspace))
create_keyspace(cassa_keyspace)
sync_table(User)
sync_table(Post)
sync_table(Channel)
sync_table(Project)
# User related tables
sync_table(UserTimeLine)
sync_table(UserFollower)
sync_table(UserFollowing)
sync_table(UserPost)
sync_table(UserProject)
# Post related Tables
sync_table(PostComment)
sync_table(PostFollower)
sync_table(PostLike)
# Channel Related Tables
sync_table(ChannelFollower)
sync_table(ChannelTimeLine)
# Project Related
sync_table(ProjectFollower)
sync_table(ProjectTimeLine)
def create():
connect()
create_keyspace('pyhackers')
sync_table(User)
sync_table(Post)
sync_table(Channel)
sync_table(Project)
# User related tables
sync_table(UserTimeLine)
sync_table(UserFollower)
sync_table(UserFollowing)
sync_table(UserPost)
sync_table(UserProject)
# Post related Tables
sync_table(PostComment)
sync_table(PostFollower)
sync_table(PostLike)
# Channel Related Tables
sync_table(ChannelFollower)
sync_table(ChannelTimeLine)
# Project Related
sync_table(ProjectFollower)
sync_table(ProjectTimeLine)
def create(cassa_keyspace='pyhackers'):
assert cassa_keyspace != '' or cassa_keyspace is not None
logging.warn("Creating/Sync'ing {}".format(cassa_keyspace))
create_keyspace(cassa_keyspace)
sync_table(User)
sync_table(Post)
sync_table(Channel)
sync_table(Project)
# User related tables
sync_table(UserTimeLine)
sync_table(UserFollower)
sync_table(UserFollowing)
sync_table(UserPost)
sync_table(UserProject)
# Post related Tables
sync_table(PostComment)
sync_table(PostFollower)
sync_table(PostLike)
# Channel Related Tables
sync_table(ChannelFollower)
sync_table(ChannelTimeLine)
# Project Related
sync_table(ProjectFollower)
sync_table(ProjectTimeLine)
def handle_noargs(self, **options):
db = options.get('database')
engine = settings.DATABASES.get(db, {}).get('ENGINE', '')
# Call regular syncdb if engine is different from ours
if engine != 'django_cassandra_engine':
return super(Command, self).handle_noargs(**options)
if django.VERSION < (1, 7):
self._import_management()
connection = connections[db]
connection.connect()
options = connection.settings_dict.get('OPTIONS', {})
replication_opts = options.get('replication', {})
keyspace = connection.settings_dict['NAME']
self.stdout.write('Creating keyspace %s..' % keyspace)
create_keyspace(keyspace, **replication_opts)
for app_name, app_models \
in connection.introspection.cql_models.iteritems():
for model in app_models:
self.stdout.write('Syncing %s.%s' % (app_name, model.__name__))
sync_table(model, create_missing_keyspace=False)
def setup_package():
try:
CASSANDRA_VERSION = int(os.environ["CASSANDRA_VERSION"])
except:
print("CASSANDRA_VERSION must be set as an environment variable. "
"One of (12, 20, 21)")
raise
if os.environ.get('CASSANDRA_TEST_HOST'):
CASSANDRA_TEST_HOST = os.environ['CASSANDRA_TEST_HOST']
else:
CASSANDRA_TEST_HOST = 'localhost'
if CASSANDRA_VERSION < 20:
protocol_version = 1
else:
protocol_version = 2
connection.setup([CASSANDRA_TEST_HOST],
protocol_version=protocol_version,
default_keyspace='cqlengine_test')
create_keyspace("cqlengine_test",
replication_factor=1,
strategy_class="SimpleStrategy")
def sync(self, alias):
engine = get_engine_from_db_alias(alias)
if engine != 'django_cassandra_engine':
raise CommandError('Database {0} is not cassandra!'.format(alias))
connection = connections[alias]
connection.connect()
options = connection.settings_dict.get('OPTIONS', {})
keyspace = connection.settings_dict['NAME']
replication_opts = options.get('replication', {})
strategy_class = replication_opts.pop('strategy_class',
'SimpleStrategy')
replication_factor = replication_opts.pop('replication_factor', 1)
self.stdout.write('Creating keyspace {0}..'.format(keyspace))
create_keyspace(keyspace, strategy_class, replication_factor,
**replication_opts)
for app_name, app_models \
in connection.introspection.cql_models.iteritems():
for model in app_models:
self.stdout.write('Syncing %s.%s' % (app_name, model.__name__))
sync_table(model)
def setUp(self):
connection.setup(['127.0.0.1'], KEYSPACE)
create_keyspace(KEYSPACE,
replication_factor=1,
strategy_class='SimpleStrategy')
sync_table(Avatar)
sync_table(Anchor)
sync_table(Message)
def cassandra_setup():
from cqlengine.management import create_table, create_keyspace
aggregated_timeline = AggregatedFeed.get_timeline_storage()
timeline = FashiolistaFeed.get_timeline_storage()
user_timeline = UserFeed.get_timeline_storage()
create_keyspace('test')
create_table(aggregated_timeline.model)
create_table(timeline.model)
create_table(user_timeline.model)
def cassandra_setup():
from cqlengine.management import create_table, create_keyspace
aggregated_timeline = AggregatedFeed.get_timeline_storage()
timeline = FashiolistaFeed.get_timeline_storage()
user_timeline = UserFeed.get_timeline_storage()
create_keyspace('test')
create_table(aggregated_timeline.model)
create_table(timeline.model)
create_table(user_timeline.model)
def setUpClass(cls):
super(BaseIfNotExistsTest, cls).setUpClass()
"""
when receiving an insert statement with 'if not exist', cassandra would
perform a read with QUORUM level. Unittest would be failed if replica_factor
is 3 and one node only. Therefore I have create a new keyspace with
replica_factor:1.
"""
create_keyspace(TestIfNotExistsModel.__keyspace__, replication_factor=1)
sync_table(TestIfNotExistsModel)
def setUpClass(cls):
super(BaseIfNotExistsTest, cls).setUpClass()
"""
when receiving an insert statement with 'if not exist', cassandra would
perform a read with QUORUM level. Unittest would be failed if replica_factor
is 3 and one node only. Therefore I have create a new keyspace with
replica_factor:1.
"""
create_keyspace(TestIfNotExistsModel.__keyspace__,
replication_factor=1)
sync_table(TestIfNotExistsModel)
def setUp(self):
keyspace = 'testkeyspace{}'.format(str(uuid.uuid1()).replace('-', ''))
self.keyspace = keyspace
clear()
# Configure cqlengine's global connection pool.
setup(['localhost'], default_keyspace=keyspace)
create_keyspace(keyspace)
for class_name, creator in self.model_classes.items():
setattr(self, class_name, creator)
#sync_table(getattr(self, class_name))
getattr(self, class_name).sync_table()
def create(cassa_key_space='pyhackers'):
assert cassa_key_space != '' or cassa_key_space is not None
logging.warn("Creating/Synchronizing {}".format(cassa_key_space))
create_keyspace(cassa_key_space)
sync_table(User)
sync_table(Post)
sync_table(Channel)
sync_table(Project)
# User related tables
sync_table(UserTimeLine)
sync_table(UserFollower)
sync_table(UserFollowing)
sync_table(UserPost)
sync_table(UserProject)
sync_table(UserDiscussion)
sync_table(UserCounter)
# Post related Tables
sync_table(PostReply)
sync_table(PostFollower)
sync_table(PostCounter)
sync_table(PostVote)
# Channel Related Tables
sync_table(ChannelFollower)
sync_table(ChannelTimeLine)
# Topic Related Topics
sync_table(Topic)
sync_table(TopicCounter)
sync_table(TopicDiscussion)
# Project Related
sync_table(ProjectFollower)
sync_table(ProjectTimeLine)
# Discussions yoo
sync_table(Discussion)
sync_table(DiscussionPost)
sync_table(DiscussionCounter)
sync_table(DiscussionFollower)
sync_table(GithubProject)
sync_table(GithubUser)
sync_table(GithubUserList)
sync_table(GithubEvent)
sync_table(Story)
sync_table(Feed)
def create_test_db(self, verbosity=1, autoclobber=False, **kwargs):
"""
Creates a test database, prompting the user for confirmation if the
database already exists. Returns the name of the test database created.
"""
# Don't import django.core.management if it isn't needed.
from django.core.management import call_command
from django.conf import settings
# If using django-nose, its runner has already set the db name
# to test_*, so restore it here so that all the models for the
# live keyspace can be found.
self.connection.connection.keyspace = \
self.connection.settings_dict['NAME']
test_database_name = self._get_test_db_name()
# Set all models keyspace to the test keyspace
self.set_models_keyspace(test_database_name)
if verbosity >= 1:
test_db_repr = ''
if verbosity >= 2:
test_db_repr = " ('%s')" % test_database_name
print("Creating test database for alias '%s'%s..." %
(self.connection.alias, test_db_repr))
self.connection.connect()
options = self.connection.settings_dict.get('OPTIONS', {})
replication_opts = options.get('replication', {})
strategy_class = replication_opts.pop('strategy_class',
'SimpleStrategy')
replication_factor = replication_opts.pop('replication_factor')
create_keyspace(self.connection.settings_dict['NAME'], strategy_class,
replication_factor, **replication_opts)
settings.DATABASES[self.connection.alias]["NAME"] = test_database_name
self.connection.settings_dict["NAME"] = test_database_name
self.connection.reconnect()
# Report syncdb messages at one level lower than that requested.
# This ensures we don't get flooded with messages during testing
# (unless you really ask to be flooded)
call_command('sync_cassandra',
verbosity=max(verbosity - 1, 0),
interactive=False,
database=self.connection.alias,
load_initial_data=False)
return test_database_name
def prepare_db():
"""
"""
global c
# init keyspace
# TODO: replication-factor:1 is just a dev config
create_keyspace(const.CQL_KEYSPACE_NAME, replication_factor=1)
# sync table
for name in find_modules(const.MODEL_PACKAGE_ROOT, recursive=True):
mod = import_string(name)
for item_name in dir(mod):
item = getattr(mod, item_name)
if type(item) == ModelMetaClass and issubclass(item, Model) and item.__name__ != 'Model':
sync_table(item)
def setup_storage(settings=None):
"""Create cassandra models."""
from caliopen.base.core import core_registry
# Make discovery happen
from caliopen.base.user.core import User
from caliopen.base.message.core.thread import Thread
from caliopen.base.message.core.message import Message
from cqlengine.management import sync_table, create_keyspace
keyspace = Configuration('global').get('cassandra.keyspace')
if not keyspace:
raise Exception('Configuration missing for cassandra keyspace')
# XXX tofix : define strategy and replication_factor in configuration
create_keyspace(keyspace, 'SimpleStrategy', 1)
for name, kls in core_registry.items():
log.info('Creating cassandra model %s' % name)
sync_table(kls._model_class)
def setup(self, force=False, throw=False):
if self._setup and not force:
return True
try:
connection.setup(self.uri, self.keyspace)
management.create_keyspace(self.keyspace, replication_factor=1, strategy_class='SimpleStrategy')
for model in self._models:
model.__keyspace__ = self.keyspace
management.sync_table(model)
except NoHostAvailable:
logger.error('Could not connect to Cassandra, expect errors.')
return False
# Note: return values are for test skipping
self._setup = True
return True
def tweets_to_cassandra(items):
from cqlengine import columns
from cqlengine import connection
from cqlengine.models import Model
from cqlengine.management import sync_table
from cqlengine.management import create_keyspace
class TweetModel(Model):
date = columns.Text(primary_key = True)
ticker = columns.Text()
host="localhost"
connection.setup(['127.0.0.1'], "cqlengine")
create_keyspace("cqlengine", "default_keyspace", 1)
sync_table(TweetModel)
for item in items:
tweet_table.create(items)
print "Number of elements in table:",TweetModel.objects.count()
def handle_noargs(self, **options):
db = options.get('database')
engine = settings.DATABASES.get(db, {}).get('ENGINE', '')
# Call regular syncdb if engine is different from ours
if engine != 'django_cassandra_engine':
return super(Command, self).handle_noargs(**options)
# Import the 'management' module within each installed app, to register
# dispatcher events.
for app_name in settings.INSTALLED_APPS:
try:
import_module('.management', app_name)
except ImportError as exc:
# This is slightly hackish. We want to ignore ImportErrors
# if the "management" module itself is missing -- but we don't
# want to ignore the exception if the management module exists
# but raises an ImportError for some reason. The only way we
# can do this is to check the text of the exception. Note that
# we're a bit broad in how we check the text, because different
# Python implementations may not use the same text.
# CPython uses the text "No module named management"
# PyPy uses "No module named myproject.myapp.management"
msg = exc.args[0]
if not msg.startswith('No module named') \
or 'management' not in msg:
raise
connection = connections[db]
connection.connect()
options = connection.settings_dict.get('OPTIONS', {})
replication_opts = options.get('replication', {})
keyspace = connection.settings_dict['NAME']
self.stdout.write('Creating keyspace %s..' % keyspace)
create_keyspace(keyspace, **replication_opts)
for app_name, app_models \
in connection.introspection.cql_models.iteritems():
for model in app_models:
self.stdout.write('Syncing %s.%s' % (app_name, model.__name__))
sync_table(model, create_missing_keyspace=False)
def create_test_db(self, verbosity=1, autoclobber=False, **kwargs):
"""
Creates a test database, prompting the user for confirmation if the
database already exists. Returns the name of the test database created.
"""
# Don't import django.core.management if it isn't needed.
from django.core.management import call_command
from django.conf import settings
test_database_name = self._get_test_db_name()
if verbosity >= 1:
test_db_repr = ''
if verbosity >= 2:
test_db_repr = " ('%s')" % test_database_name
print("Creating test database for alias '%s'%s..." % (
self.connection.alias, test_db_repr))
self.connection.connect()
options = self.connection.settings_dict.get('OPTIONS', {})
replication_opts = options.get('replication', {})
create_keyspace(self.connection.settings_dict['NAME'],
**replication_opts)
settings.DATABASES[self.connection.alias]["NAME"] = test_database_name
self.connection.settings_dict["NAME"] = test_database_name
self.connection.reconnect()
# Report syncdb messages at one level lower than that requested.
# This ensures we don't get flooded with messages during testing
# (unless you really ask to be flooded)
call_command(
'sync_cassandra',
verbosity=max(verbosity - 1, 0),
interactive=False,
database=self.connection.alias,
load_initial_data=False
)
return test_database_name
def handle_noargs(self, **options):
# Import the 'management' module within each installed app, to register
# dispatcher events.
for app_name in settings.INSTALLED_APPS:
try:
import_module('.management', app_name)
except ImportError as exc:
# This is slightly hackish. We want to ignore ImportErrors
# if the "management" module itself is missing -- but we don't
# want to ignore the exception if the management module exists
# but raises an ImportError for some reason. The only way we
# can do this is to check the text of the exception. Note that
# we're a bit broad in how we check the text, because different
# Python implementations may not use the same text.
# CPython uses the text "No module named management"
# PyPy uses "No module named myproject.myapp.management"
msg = exc.args[0]
if not msg.startswith('No module named') \
or 'management' not in msg:
raise
db = options.get('database')
connection = connections[db]
connection.connect()
options = connection.settings_dict.get('OPTIONS', {})
replication_opts = options.get('replication', {})
keyspace = connection.settings_dict['NAME']
self.stdout.write('Creating keyspace %s..' % keyspace)
create_keyspace(keyspace, **replication_opts)
apps = models.get_apps()
for app in apps:
app_models = get_cql_models(app)
for model in app_models:
self.stdout.write('Syncing %s.%s' %
(app.__name__, model.__name__))
sync_table(model, create_missing_keyspace=False)
def get_current_revision(self):
"""Return the current revision, usually that which is present
in the ``alembic_version`` table in the database.
If this :class:`.MigrationContext` was configured in "offline"
mode, that is with ``as_sql=True``, the ``starting_rev``
parameter is returned instead, if any.
"""
if self.as_sql:
return self._start_from_rev
else:
if self._start_from_rev:
raise util.CommandError(
"Can't specify current_rev to context "
"when using a database connection")
if 'cqlengine' == self.dialect.name:
from cqlengine.exceptions import CQLEngineException
# Try to load the data from the DB.
# if the CF doesn't exist, create it, if the row doesn't exist
# initialize it.
try:
return self._version.get(key='alembic').version_num
except CQLEngineException, e:
from cqlengine import management, models
if 'Keyspace' in e.message and 'does not exist' in e.message:
management.create_keyspace(models.DEFAULT_KEYSPACE)
management.sync_table(self._version)
self._version.create(key='alembic')
return self.get_current_revision()
elif "Bad Request: unconfigured columnfamily alembic_version" in e.message:
management.sync_table(self._version)
self._version.create(key='alembic')
return self.get_current_revision()
elif type(e).__name__ == 'DoesNotExist':
self._version.create(key='alembic')
return self.get_current_revision()
else:
def setUp(self):
app = flask.Flask(__name__)
app.config['TESTING'] = True
app.config['CQLENGINE_HOSTS'] = 'localhost'
app.config['CQLENGINE_PORT'] = '9042'
app.config['CQLENGINE_DEFAULT_KEYSPACE'] = 'testkeyspace{}'.format(str(uuid.uuid1()).replace('-', ''))
cqlengine.CQLEngine(app)
self.Todo = make_todo_model()
@app.route('/')
def index():
return '\n'.join(x.title for x in self.Todo.objects)
@app.route('/add', methods=['POST'])
def add():
form = flask.request.form
todo = self.Todo.create(title=form['title'], text=form['text'])
return 'added'
create_keyspace(app.config['CQLENGINE_DEFAULT_KEYSPACE'])
self.Todo.sync_table()
self.app = app
example_type = columns.Integer(index=True)
created_at = columns.DateTime()
description = columns.Text(required=False)
#next, setup the connection to your cassandra server(s) and the default keyspace...
from cqlengine import connection
connection.setup(['127.0.0.1'], "cqlengine")
# or if you're still on cassandra 1.2
#connection.setup(['127.0.0.1'], "cqlengine", protocol_version=1)
# create your keyspace. This is, in general, not what you want in production
# see https://cassandra.apache.org/doc/cql3/CQL.html#createKeyspaceStmt for options
from cqlengine.management import create_keyspace
create_keyspace("cqlengine", "SimpleStrategy", 1)
#...and create your CQL table
from cqlengine.management import sync_table
sync_table(ExampleModel)
#now we can create some rows:
em1 = ExampleModel.create(example_type=0,
description="example1",
created_at=datetime.now())
em2 = ExampleModel.create(example_type=0,
description="example2",
created_at=datetime.now())
em3 = ExampleModel.create(example_type=0,
description="example3",
created_at=datetime.now())
from cqlengine.management import sync_table, create_keyspace
# Setup connection
from cqlengine import connection
# Kafka broker
kafka_brokers_list = ["localhost:9092"] # We can put multiple brokers here.
group_id = "meetup_event_stream" # group id of topics
# Kafka Consumer
consumer = KafkaConsumer("eventstream", group_id=group_id)
# Connect to the meetup keyspace on our cluster running at 127.0.0.1
connection.setup(['127.0.0.1'], "meetup")
# create keyspace
# keyspace name, keyspace replication strategy, replication factor
create_keyspace('meetup', 'SimpleStrategy', 1)
# Sync your model with your cql table
sync_table(Eventstream)
def write_to_cassandra(**kwargs):
"""
Write the data to cassandra.
"""
try:
Eventstream.create(
status=get_dict_val(kwargs, 'status'),
venue_name=get_dict_val(kwargs, 'venue_name'),
venue_lon=get_dict_val(kwargs, 'venue_lon'),
venue_lat=get_dict_val(kwargs, 'venue_lat'),
venue_id=get_dict_val(kwargs, 'venue_id'),
def test_create_succeeeds(self):
management.create_keyspace('test_keyspace')
management.delete_keyspace('test_keyspace')
example_id = columns.UUID(primary_key=True, default=uuid.uuid4)
example_type = columns.Integer(index=True)
created_at = columns.DateTime()
description = columns.Text(required=False)
#next, setup the connection to your cassandra server(s) and the default keyspace...
from cqlengine import connection
connection.setup(['127.0.0.1'], "cqlengine")
# or if you're still on cassandra 1.2
#connection.setup(['127.0.0.1'], "cqlengine", protocol_version=1)
# create your keyspace. This is, in general, not what you want in production
# see https://cassandra.apache.org/doc/cql3/CQL.html#createKeyspaceStmt for options
from cqlengine.management import create_keyspace
create_keyspace("cqlengine", "SimpleStrategy", 1)
#...and create your CQL table
from cqlengine.management import sync_table
sync_table(ExampleModel)
#now we can create some rows:
em1 = ExampleModel.create(example_type=0, description="example1", created_at=datetime.now())
em2 = ExampleModel.create(example_type=0, description="example2", created_at=datetime.now())
em3 = ExampleModel.create(example_type=0, description="example3", created_at=datetime.now())
em4 = ExampleModel.create(example_type=0, description="example4", created_at=datetime.now())
em5 = ExampleModel.create(example_type=1, description="example5", created_at=datetime.now())
em6 = ExampleModel.create(example_type=1, description="example6", created_at=datetime.now())
em7 = ExampleModel.create(example_type=1, description="example7", created_at=datetime.now())
em8 = ExampleModel.create(example_type=1, description="example8", created_at=datetime.now())
def setUp(self):
keyspace = 'testkeyspace{}'.format(str(uuid.uuid1()).replace('-', ''))
self.keyspace = keyspace
# Configure cqlengine's global connection pool.
setup(['localhost'], default_keyspace=keyspace)
create_keyspace(keyspace)
print message.value
"""
# Custom Code Imports
#from RsvpStream import Rsvpstream # Cassandra Model
from utils import get_dict_val, load_json, datetime_from_epoch # util functions
# Managing schemas
from cqlengine.management import sync_table, create_keyspace
# Setup connection
from cqlengine import connection
# Connect to the meetup keyspace on our cluster running at 127.0.0.1
connection.setup(['127.0.0.1'], "meetup")
# create keyspace
# keyspace name, keyspace replication strategy, replication factor
create_keyspace('meetup', 'SimpleStrategy', 1)
# Sync your model with your cql table
"""
sync_table(Rsvpstream)
"""
def write_to_cassandra(**kwargs):
"""
Write the data to cassandra.
"""
try:
Rsvpstream.create(
venue_name=get_dict_val(kwargs, 'venue_name'),
venue_lon=get_dict_val(kwargs, 'venue_lon'),
venue_lat=get_dict_val(kwargs, 'venue_lat'),
def test_create_succeeeds(self):
management.create_keyspace('test_keyspace', strategy_class="SimpleStrategy", replication_factor=1)
management.delete_keyspace('test_keyspace')
def connect():
setup("127.0.0.1", "tutorial")
management.create_keyspace("tutorial", replication_factor=1)
