예제 #1
0
    def test_sync_table_works_with_primary_keys_only_tables(self):

        # This is "create table":

        sync_table(PrimaryKeysOnlyModel)

        # let's make sure settings persisted correctly:

        assert PrimaryKeysOnlyModel.__compaction__ == LeveledCompactionStrategy
        # blows up with DoesNotExist if table does not exist
        table_settings = management.get_table_settings(PrimaryKeysOnlyModel)
        # let make sure the flag we care about

        assert LeveledCompactionStrategy in table_settings.options['compaction_strategy_class']


        # Now we are "updating" the table:

        # setting up something to change
        PrimaryKeysOnlyModel.__compaction__ = SizeTieredCompactionStrategy

        # primary-keys-only tables do not create entries in system.schema_columns
        # table. Only non-primary keys are added to that table.
        # Our code must deal with that eventuality properly (not crash)
        # on subsequent runs of sync_table (which runs get_fields internally)
        get_fields(PrimaryKeysOnlyModel)
        sync_table(PrimaryKeysOnlyModel)

        table_settings = management.get_table_settings(PrimaryKeysOnlyModel)
        assert SizeTieredCompactionStrategy in table_settings.options['compaction_strategy_class']
예제 #2
0
    def test_sync_table_works_with_primary_keys_only_tables(self):

        # This is "create table":

        sync_table(PrimaryKeysOnlyModel)

        # let's make sure settings persisted correctly:

        assert PrimaryKeysOnlyModel.__compaction__ == LeveledCompactionStrategy
        # blows up with DoesNotExist if table does not exist
        table_settings = management.get_table_settings(PrimaryKeysOnlyModel)
        # let make sure the flag we care about

        assert LeveledCompactionStrategy in table_settings.options[
            'compaction_strategy_class']

        # Now we are "updating" the table:

        # setting up something to change
        PrimaryKeysOnlyModel.__compaction__ = SizeTieredCompactionStrategy

        # primary-keys-only tables do not create entries in system.schema_columns
        # table. Only non-primary keys are added to that table.
        # Our code must deal with that eventuality properly (not crash)
        # on subsequent runs of sync_table (which runs get_fields internally)
        get_fields(PrimaryKeysOnlyModel)
        sync_table(PrimaryKeysOnlyModel)

        table_settings = management.get_table_settings(PrimaryKeysOnlyModel)
        assert SizeTieredCompactionStrategy in table_settings.options[
            'compaction_strategy_class']
예제 #3
0
    def test_add_column(self):
        create_table(FirstModel)
        fields = get_fields(FirstModel)

        # this should contain the second key
        self.assertEqual(len(fields), 2)
        # get schema
        create_table(SecondModel)

        fields = get_fields(FirstModel)
        self.assertEqual(len(fields), 3)

        create_table(ThirdModel)
        fields = get_fields(FirstModel)
        self.assertEqual(len(fields), 4)

        create_table(FourthModel)
        fields = get_fields(FirstModel)
        self.assertEqual(len(fields), 4)
def verify(*models, **kwargs):
    ignore_extra = kwargs.get('ignore_extra', {})
    results = {}
    by_keyspace = {}
    by_cf = {}
    results = {}
    for model in models:
        ks_name = model._get_keyspace()
        try:
            by_keyspace[ks_name].add(model)
        except KeyError:
            by_keyspace[ks_name] = set([model])
        cf_name = model.column_family_name(include_keyspace=False)
        by_cf[cf_name] = model
        results[model] = VerifyResult(model)

    for keyspace, models in by_keyspace.items():
        query_result = cqlengine.connection.get_session().execute(
        "SELECT columnfamily_name, key_aliases, key_validator, column_aliases, comparator from system.schema_columnfamilies WHERE keyspace_name = %(ks_name)s",
                {'ks_name': ks_name})
        tables = {}
        for result in query_result:
            columnfamily_name = result['columnfamily_name']
            partition_keys = result['key_aliases']
            partition_key_types = result['key_validator']
            primary_keys = result['column_aliases']
            primary_key_types = result['comparator']
            partition_keys = json.loads(partition_keys)
            if len(partition_keys) > 1:
                partition_key_types = partition_key_types[len('org.apache.cassandra.db.marshal.CompositeType('):-1].split(',')[:len(partition_keys)]
            else:
                partition_key_types = [partition_key_types]
            primary_keys = json.loads(primary_keys)
            primary_key_types = primary_key_types[len('org.apache.cassandra.db.marshal.CompositeType('):].split(',')[:len(primary_keys)]
            item = {
                'cf': columnfamily_name,
                'partition_keys': partition_keys,
                'partition_key_types': partition_key_types,
                'primary_keys': primary_keys,
                'primary_key_types': primary_key_types
            }
            tables[columnfamily_name] = item
        for model in models:
            cf_name = model.column_family_name(include_keyspace=False)
            db_field_names = {col.db_field_name: col for name, col in model._columns.items()}
            result = results[model]
            # Check that model's cf is in db's tables.
            if cf_name not in tables:
                result.is_missing = True
            else:
                table_info = tables[cf_name]
                fields = get_fields(model)
                fields = {field.name: field.type for field in fields}
                for name, field_type in fields.iteritems():
                    # If field is missing, that's an error.
                    if name not in db_field_names:
                        result.extra.add(name)
                    # If field is present, check the type.
                    else:
                        col = db_field_names[name]
                        if isinstance(col, columns.Map):
                            if not field_type.startswith('org.apache.cassandra.db.marshal.MapType'):
                                result.different.add(col.column_name)
                        elif isinstance(col, columns.List):
                            if not field_type.startswith('org.apache.cassandra.db.marshal.ListType'):
                                result.different.add(col.column_name)
                        elif isinstance(col, columns.Set):
                            if not field_type.startswith('org.apache.cassandra.db.marshal.SetType'):
                                result.different.add(col.column_name)
                        else:
                            local_metadata = _type_to_metadata(col.db_type)
                            if local_metadata != field_type:
                                result.different.add(col.column_name)
                for name, kind in zip(table_info['partition_keys'], table_info['partition_key_types']):
                    if name not in db_field_names:
                        result.extra.add(name)
                    else:
                        col = db_field_names[name]
                        local_metadata = _type_to_metadata(col.db_type)
                        if local_metadata != kind:
                            result.different.add(col.column_name)
                for name, kind in zip(table_info['primary_keys'], table_info['primary_key_types']):
                    if name not in db_field_names:
                        result.extra.add(name)
                    else:
                        col = db_field_names[name]
                        local_metadata = _type_to_metadata(col.db_type)
                        if col.clustering_order == 'desc':
                            local_metadata = u'org.apache.cassandra.db.marshal.ReversedType({})'.format(local_metadata)
                        if local_metadata != kind:
                            result.different.add(col.column_name)

                for name, col in db_field_names.items():
                    # Handle primary keys from table-level data.
                    if col.primary_key:
                        if col.partition_key:
                            if name not in table_info['partition_keys']:
                                result.missing.add(col.column_name)
                            else:
                                local_metadata = _type_to_metadata(col.db_type)
                                i = table_info['partition_keys'].index(name)
                                if local_metadata != table_info['partition_key_types'][i]:
                                    result.different.add(col.column_name)
                        else:
                            if name not in table_info['primary_keys']:
                                result.missing.add(col.column_name)
                            else:
                                local_metadata = _type_to_metadata(col.db_type)
                                if col.clustering_order == 'desc':
                                    local_metadata = u'org.apache.cassandra.db.marshal.ReversedType({})'.format(local_metadata)
                                i = table_info['primary_keys'].index(name)
                                if local_metadata != table_info['primary_key_types'][i]:
                                    result.different.add(col.column_name)

                    # Primary keys are not listed in fields.
                    if not col.primary_key and name not in fields:
                        result.missing.add(col.column_name)
        for cf in tables:
            if cf not in by_cf and cf not in ignore_extra:
                result = VerifyResult(cf)
                result.is_extra = True
                results[cf] = result
        model_indexes = {}
        for model in models:
            this_model_indexes = {col.db_field_name: col for name, col in model._columns.items() if col.index}
            if this_model_indexes:
                model_indexes[model.column_family_name(include_keyspace=False)] = this_model_indexes
        query_results = cqlengine.connection.get_session().execute(
            "SELECT index_name from system.\"IndexInfo\" WHERE table_name=%(table_name)s",
            {'table_name': model._get_keyspace()})
        cassandra_indexes = {}
        for result_dict in query_results:
            idx = result_dict['index_name']
            try:
                cf, index_name = idx.split('.')
                look_for = 'index_{}_'.format(cf)
                index_name = index_name[len(look_for):]
            except ValueError:
                cf = None
                index_name = None
            if cf:
                try:
                    cassandra_indexes[cf].add(index_name)
                except KeyError:
                    cassandra_indexes[cf] = set([index_name])
        for cf, index_names in cassandra_indexes.items():
            if cf not in model_indexes:
                if cf not in by_cf:
                    result = VerifyResult(cf)
                    result.is_extra = True
                    results[cf] = result
                else:
                    model = by_cf[cf]
                    result = results[model]
                    result.extra_indexes.add(index_name)
            else:
                this_model_indexes = model_indexes[cf]
                if index_name not in this_model_indexes:
                    model = by_cf[cf]
                    result = results[model]
                    result.extra_indexes.add(index_name)
        for cf, this_model_indexes in model_indexes.items():
            for index_name in this_model_indexes.keys():
                if cf not in cassandra_indexes or index_name not in cassandra_indexes[cf]:
                    model = by_cf[cf]
                    result = results[model]
                    result.missing_indexes.add(index_name)

    results = {model: result for model, result in results.items() if result.has_errors()}

    return results.values()