Exemplo n.º 1
0
def search_data_hierarchy_fr(list_geonameid):
    """[summary]

    Args:
        list_geonameid ([type]): [description]

    Returns:
        [type]: [description]
    """
    bool_h = True
    list_geonameid = list_to_str(list_geonameid)
    hierarchy = []

    while bool_h is True:  #recherche de la hiérarchie
        query = 'SELECT parentId FROM cassandra.hierarchy where childId IN '
        row_parent_id = connection.execute(query + '(' + list_geonameid +
                                           ') ALLOW FILTERING')
        row_parent_id = row_parent_id.one()
        if row_parent_id is not None:
            row_parent_id = row_parent_id['parentid']
            row_parent_id = str(row_parent_id)
            list_geonameid = row_parent_id
            hierarchy.append(row_parent_id)
        else:
            bool_h = False

    liste_hierarchy = []

    #itération 1 par 1 pour que la hiérarchie soit enregitrée de la plus petite à la plus grande
    for hier in hierarchy:
        rows = connection.execute(
            'SELECT * FROM cassandra.geo_name WHERE geonameid=' + hier)
        liste_hierarchy.append(rows.one())

    return liste_hierarchy
Exemplo n.º 2
0
    def trim(self, key, length, batch_interface=None):
        '''
        trim using Cassandra's tombstones black magic
        retrieve the WRITETIME of the last item we want to keep
        then delete everything written after that

        this is still pretty inefficient since it needs to retrieve
        length amount of items

        WARNING: since activities created using Batch share the same timestamp
        trim can trash up to (batch_size - 1) more activities than requested

        '''
        query = "SELECT WRITETIME(%s) as wt FROM %s.%s WHERE feed_id='%s' ORDER BY activity_id DESC LIMIT %s;"
        trim_col = [c for c in self.model._columns.keys(
        ) if c not in self.model._primary_keys.keys()][0]
        parameters = (
            trim_col, self.model._get_keyspace(), self.column_family_name, key, length + 1)
        results = execute(query % parameters)
        
        # compatibility with both cassandra driver 2.7 and 3.0
        results_length = len(results.current_rows) if hasattr(results, 'current_rows') else len(results)
        if results_length < length:
            return
        trim_ts = (results[-1]['wt'] + results[-2]['wt']) // 2
        delete_query = "DELETE FROM %s.%s USING TIMESTAMP %s WHERE feed_id='%s';"
        delete_params = (
            self.model._get_keyspace(), self.column_family_name, trim_ts, key)
        execute(delete_query % delete_params)
Exemplo n.º 3
0
 def test_extra_field(self):
     drop_table(self.TestModel)
     sync_table(self.TestModel)
     self.TestModel.create()
     execute("ALTER TABLE {0} add blah int".format(
         self.TestModel.column_family_name(include_keyspace=True)))
     self.TestModel.objects().all()
Exemplo n.º 4
0
def create_keyspace():
    logger.info("create_keyspace(): creating keyspace %s",
                settings.CASSANDRA_CONNECTION['KEYSPACE'])
    connection.execute("CREATE KEYSPACE IF NOT EXISTS {} WITH REPLICATION = {}".format(
        settings.CASSANDRA_CONNECTION['KEYSPACE'],
        settings.CASSANDRA_CONNECTION['KEYSPACE_REPLICATION']
    ))
    def test_insert_statement_execute(self):
        """
        Test to verify the execution of BaseCQLStatements using connection.execute

        @since 3.10
        @jira_ticket PYTHON-505
        @expected_result inserts a row in C*, updates the rows and then deletes
        all the rows using BaseCQLStatements

        @test_category data_types:object_mapper
        """
        partition = uuid4()
        cluster = 1
        self._insert_statement(partition, cluster)

        # Verifying update statement
        where = [WhereClause('partition', EqualsOperator(), partition),
                 WhereClause('cluster', EqualsOperator(), cluster)]

        st = UpdateStatement(self.table_name, where=where)
        st.add_assignment(Column(db_field='count'), 2)
        st.add_assignment(Column(db_field='text'), "text_for_db_update")
        st.add_assignment(Column(db_field='text_set'), set(("foo_update", "bar_update")))
        st.add_assignment(Column(db_field='text_list'), ["foo_update", "bar_update"])
        st.add_assignment(Column(db_field='text_map'), {"foo": '3', "bar": '4'})

        execute(st)
        self._verify_statement(st)

        # Verifying delete statement
        execute(DeleteStatement(self.table_name, where=where))
        self.assertEqual(TestQueryUpdateModel.objects.count(), 0)
Exemplo n.º 6
0
    def trim(self, key, length, batch_interface=None):
        '''
        trim using Cassandra's tombstones black magic
        retrieve the WRITETIME of the last item we want to keep
        then delete everything written after that

        this is still pretty inefficient since it needs to retrieve
        length amount of items

        WARNING: since activities created using Batch share the same timestamp
        trim can trash up to (batch_size - 1) more activities than requested

        '''
        query = "SELECT WRITETIME(%s) as wt FROM %s.%s WHERE feed_id='%s' ORDER BY activity_id DESC LIMIT %s;"
        trim_col = [c for c in self.model._columns.keys(
        ) if c not in self.model._primary_keys.keys()][0]
        parameters = (
            trim_col, self.model._get_keyspace(), self.column_family_name, key, length + 1)
        results = execute(query % parameters)
        
        # compatibility with both cassandra driver 2.7 and 3.0
        results_length = len(results.current_rows) if hasattr(results, 'current_rows') else len(results)
        if results_length < length:
            return
        trim_ts = (results[-1]['wt'] + results[-2]['wt']) // 2
        delete_query = "DELETE FROM %s.%s USING TIMESTAMP %s WHERE feed_id='%s';"
        delete_params = (
            self.model._get_keyspace(), self.column_family_name, trim_ts, key)
        execute(delete_query % delete_params)
Exemplo n.º 7
0
def get_avg_x_speed():

    #with UDF
    connection.execute(CREATE_GET_X_SPEED_FUNCTION_QUERY.format(DB_NAME))
    return list(
        connection.execute(
            AVG_X_SPEED_QUERY.format(DB_NAME)).current_rows[0].values())[0]
def create_materialized_view(model):
    """materialized_view must has base_model. Must with all base_model primary keys as primary key."""
    table_cql = _get_create_table(model)
    """
    cql example:
    CREATE TABLE rolltrek.course2 ("course_id" uuid , "user_id" uuid , "created_at" timestamp , PRIMARY KEY (("course_id"), "user_id")) WITH CLUSTERING ORDER BY ("user_id" ASC)
    """
    m = re.match(r'CREATE TABLE (\S+) \((.*?)(PRIMARY KEY.*$)', table_cql)
    name = m[1]
    end = m[3].replace(')) WITH CLUSTERING', ') WITH CLUSTERING')
    # resolve columnNames
    t = m[2]
    columnNames = []  # maybe wrapped with quota
    for t2 in t.split(','):
        t2 = t2.strip()
        if t2:
            columnNames.append(t2.split(' ')[0])
    # join cql
    select_part = ', '.join(columnNames)
    where_part = ' AND '.join(
        ['%s IS NOT NULL' % (name) for name in columnNames])
    base_table_name = model.base_model.column_family_name()
    cql = """
CREATE MATERIALIZED VIEW %s AS
SELECT %s FROM %s WHERE %s
%s;
    """ % (name, select_part, base_table_name, where_part, end)
    connection.execute(cql)
Exemplo n.º 9
0
    def __create_keyspace(name, durable_writes, strategy_class, strategy_options, connection=None):
        cluster = get_cluster(connection)

        if name not in cluster.metadata.keyspaces:
            log.info(format_log_context("Creating keyspace %s", connection=connection), name)
            ks_meta = metadata.KeyspaceMetadata(name, durable_writes, strategy_class, strategy_options)
            execute(ks_meta.as_cql_query(), connection=connection)
        else:
            log.info(format_log_context("Not creating keyspace %s because it already exists", connection=connection), name)
Exemplo n.º 10
0
    def __create_keyspace(name, durable_writes, strategy_class, strategy_options, connection=None):
        cluster = get_cluster(connection)

        if name not in cluster.metadata.keyspaces:
            log.info(format_log_context("Creating keyspace %s", connection=connection), name)
            ks_meta = metadata.KeyspaceMetadata(name, durable_writes, strategy_class, strategy_options)
            execute(ks_meta.as_cql_query(), connection=connection)
        else:
            log.info(format_log_context("Not creating keyspace %s because it already exists", connection=connection), name)
    def sync(self, alias):
        super(Command, self).sync(alias)

        if hasattr(settings, 'EPO_NORMAL_CASSANDRA_ACCESS_SETTINGS'):
            user = settings.EPO_NORMAL_CASSANDRA_ACCESS_SETTINGS['USER']
            connection = connections[alias]
            keyspace = connection.settings_dict['NAME']
            execute(GRANT_PATTERN % ('select', keyspace, user))
            execute(GRANT_PATTERN % ('modify', keyspace, user))
Exemplo n.º 12
0
def _update_options(model, connection=None):
    """Updates the table options for the given model if necessary.

    :param model: The model to update.
    :param connection: Name of the connection to use

    :return: `True`, if the options were modified in Cassandra,
        `False` otherwise.
    :rtype: bool
    """
    ks_name = model._get_keyspace()
    msg = format_log_context("Checking %s for option differences",
                             keyspace=ks_name,
                             connection=connection)
    log.debug(msg, model)
    model_options = model.__options__ or {}

    table_meta = _get_table_metadata(model, connection=connection)
    # go to CQL string first to normalize meta from different versions
    existing_option_strings = set(
        table_meta._make_option_strings(table_meta.options))
    existing_options = _options_map_from_strings(existing_option_strings)
    model_option_strings = metadata.TableMetadataV3._make_option_strings(
        model_options)
    model_options = _options_map_from_strings(model_option_strings)

    update_options = {}
    for name, value in model_options.items():
        try:
            existing_value = existing_options[name]
        except KeyError:
            msg = format_log_context(
                "Invalid table option: '%s'; known options: %s",
                keyspace=ks_name,
                connection=connection)
            raise KeyError(msg % (name, existing_options.keys()))
        if isinstance(existing_value, six.string_types):
            if value != existing_value:
                update_options[name] = value
        else:
            try:
                for k, v in value.items():
                    if existing_value[k] != v:
                        update_options[name] = value
                        break
            except KeyError:
                update_options[name] = value

    if update_options:
        options = ' AND '.join(
            metadata.TableMetadataV3._make_option_strings(update_options))
        query = "ALTER TABLE {0} WITH {1}".format(model.column_family_name(),
                                                  options)
        execute(query, connection=connection)
        return True

    return False
Exemplo n.º 13
0
def _sync_type(ks_name, type_model, omit_subtypes=None, connection=None):
    syncd_sub_types = omit_subtypes or set()
    for field in type_model._fields.values():
        udts = []
        columns.resolve_udts(field, udts)
        for udt in [u for u in udts if u not in syncd_sub_types]:
            _sync_type(ks_name, udt, syncd_sub_types, connection=connection)
            syncd_sub_types.add(udt)

    type_name = type_model.type_name()
    type_name_qualified = "%s.%s" % (ks_name, type_name)

    cluster = get_cluster(connection)

    keyspace = cluster.metadata.keyspaces[ks_name]
    defined_types = keyspace.user_types

    if type_name not in defined_types:
        log.debug(format_log_context("sync_type creating new type %s", keyspace=ks_name, connection=connection),
                  type_name_qualified)
        cql = get_create_type(type_model, ks_name)
        execute(cql, connection=connection)
        cluster.refresh_user_type_metadata(ks_name, type_name)
        type_model.register_for_keyspace(ks_name, connection=connection)
    else:
        type_meta = defined_types[type_name]
        defined_fields = type_meta.field_names
        model_fields = set()
        for field in type_model._fields.values():
            model_fields.add(field.db_field_name)
            if field.db_field_name not in defined_fields:
                execute("ALTER TYPE {0} ADD {1}".format(type_name_qualified, field.get_column_def()),
                        connection=connection)
            else:
                field_type = type_meta.field_types[defined_fields.index(field.db_field_name)]
                if field_type != field.db_type:
                    msg = format_log_context(
                        'Existing user type {0} has field "{1}" with a type ({2}) differing from the model user type ({3}).'
                        ' UserType should be updated.', keyspace=ks_name, connection=connection)
                    msg = msg.format(type_name_qualified, field.db_field_name, field_type, field.db_type)
                    warnings.warn(msg)
                    log.warning(msg)

        type_model.register_for_keyspace(ks_name, connection=connection)

        if len(defined_fields) == len(model_fields):
            log.info(
                format_log_context("Type %s did not require synchronization", keyspace=ks_name, connection=connection),
                type_name_qualified)
            return

        db_fields_not_in_model = model_fields.symmetric_difference(defined_fields)
        if db_fields_not_in_model:
            msg = format_log_context("Type %s has fields not referenced by model: %s", keyspace=ks_name,
                                     connection=connection)
            log.info(msg, type_name_qualified, db_fields_not_in_model)
Exemplo n.º 14
0
def create_keyspace(name,
                    strategy_class,
                    replication_factor,
                    durable_writes=True,
                    **replication_values):
    """
    *Deprecated - use :func:`create_keyspace_simple` or :func:`create_keyspace_network_topology` instead*

    Creates a keyspace

    If the keyspace already exists, it will not be modified.

    **This function should be used with caution, especially in production environments.
    Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).**

    *There are plans to guard schema-modifying functions with an environment-driven conditional.*

    :param str name: name of keyspace to create
    :param str strategy_class: keyspace replication strategy class (:attr:`~.SimpleStrategy` or :attr:`~.NetworkTopologyStrategy`
    :param int replication_factor: keyspace replication factor, used with :attr:`~.SimpleStrategy`
    :param bool durable_writes: Write log is bypassed if set to False
    :param \*\*replication_values: Additional values to ad to the replication options map
    """
    if not _allow_schema_modification():
        return

    msg = "Deprecated. Use create_keyspace_simple or create_keyspace_network_topology instead"
    warnings.warn(msg, DeprecationWarning)
    log.warning(msg)

    cluster = get_cluster()

    if name not in cluster.metadata.keyspaces:
        # try the 1.2 method
        replication_map = {
            'class': strategy_class,
            'replication_factor': replication_factor
        }
        replication_map.update(replication_values)
        if strategy_class.lower() != 'simplestrategy':
            # Although the Cassandra documentation states for `replication_factor`
            # that it is "Required if class is SimpleStrategy; otherwise,
            # not used." we get an error if it is present.
            replication_map.pop('replication_factor', None)

        query = """
        CREATE KEYSPACE {}
        WITH REPLICATION = {}
        """.format(name,
                   json.dumps(replication_map).replace('"', "'"))

        if strategy_class != 'SimpleStrategy':
            query += " AND DURABLE_WRITES = {}".format(
                'true' if durable_writes else 'false')

        execute(query)
Exemplo n.º 15
0
def remove(timestamp, entity_name, dicted=False):
    deleted = connection.execute("select * from {0}.{3} where date='{1}' and time='{2}';"\
    .format(DB_NAME, timestamp.strftime(DATE_PATTERN), timestamp.strftime(TIME_PATTERN), entity_name))[0]

    connection.execute("delete from {0}.{3} where date='{1}' and time='{2}';"\
    .format(DB_NAME, timestamp.strftime(DATE_PATTERN), timestamp.strftime(TIME_PATTERN), entity_name))

    if not dicted:
        return namedtuple('Struct', deleted.keys())(*deleted.values())
    return deleted
Exemplo n.º 16
0
def index():
    execute(
        """
        INSERT INTO mykeyspa.users (name, credits, user_id)
        VALUES (%s, %s, %s)
        """,
        ("John O'Reilly", 42, 123)
    )

    return 'Hello Bunny my precious!!'
Exemplo n.º 17
0
def _create_keyspace(name, durable_writes, strategy_class, strategy_options):
    if not _allow_schema_modification():
        return

    cluster = get_cluster()

    if name not in cluster.metadata.keyspaces:
        log.info("Creating keyspace %s ", name)
        ks_meta = metadata.KeyspaceMetadata(name, durable_writes, strategy_class, strategy_options)
        execute(ks_meta.as_cql_query())
    else:
        log.info("Not creating keyspace %s because it already exists", name)
Exemplo n.º 18
0
def _create_keyspace(name, durable_writes, strategy_class, strategy_options):
    if not _allow_schema_modification():
        return

    cluster = get_cluster()

    if name not in cluster.metadata.keyspaces:
        log.info("Creating keyspace %s ", name)
        ks_meta = metadata.KeyspaceMetadata(name, durable_writes,
                                            strategy_class, strategy_options)
        execute(ks_meta.as_cql_query())
    else:
        log.info("Not creating keyspace %s because it already exists", name)
Exemplo n.º 19
0
def _sync_type(ks_name, type_model, omit_subtypes=None, connection=None):

    syncd_sub_types = omit_subtypes or set()
    for field in type_model._fields.values():
        udts = []
        columns.resolve_udts(field, udts)
        for udt in [u for u in udts if u not in syncd_sub_types]:
            _sync_type(ks_name, udt, syncd_sub_types, connection=connection)
            syncd_sub_types.add(udt)

    type_name = type_model.type_name()
    type_name_qualified = "%s.%s" % (ks_name, type_name)

    cluster = get_cluster(connection)

    keyspace = cluster.metadata.keyspaces[ks_name]
    defined_types = keyspace.user_types

    if type_name not in defined_types:
        log.debug(format_log_context("sync_type creating new type %s", keyspace=ks_name, connection=connection), type_name_qualified)
        cql = get_create_type(type_model, ks_name)
        execute(cql, connection=connection)
        cluster.refresh_user_type_metadata(ks_name, type_name)
        type_model.register_for_keyspace(ks_name, connection=connection)
    else:
        type_meta = defined_types[type_name]
        defined_fields = type_meta.field_names
        model_fields = set()
        for field in type_model._fields.values():
            model_fields.add(field.db_field_name)
            if field.db_field_name not in defined_fields:
                execute("ALTER TYPE {0} ADD {1}".format(type_name_qualified, field.get_column_def()), connection=connection)
            else:
                field_type = type_meta.field_types[defined_fields.index(field.db_field_name)]
                if field_type != field.db_type:
                    msg = format_log_context('Existing user type {0} has field "{1}" with a type ({2}) differing from the model user type ({3}).'
                                  ' UserType should be updated.', keyspace=ks_name, connection=connection)
                    msg = msg.format(type_name_qualified, field.db_field_name, field_type, field.db_type)
                    warnings.warn(msg)
                    log.warning(msg)

        type_model.register_for_keyspace(ks_name, connection=connection)

        if len(defined_fields) == len(model_fields):
            log.info(format_log_context("Type %s did not require synchronization", keyspace=ks_name, connection=connection), type_name_qualified)
            return

        db_fields_not_in_model = model_fields.symmetric_difference(defined_fields)
        if db_fields_not_in_model:
            msg = format_log_context("Type %s has fields not referenced by model: %s", keyspace=ks_name, connection=connection)
            log.info(msg, type_name_qualified, db_fields_not_in_model)
    def _insert_statement(self, partition, cluster):
        # Verifying insert statement
        st = InsertStatement(self.table_name)
        st.add_assignment(Column(db_field='partition'), partition)
        st.add_assignment(Column(db_field='cluster'), cluster)

        st.add_assignment(Column(db_field='count'), 1)
        st.add_assignment(Column(db_field='text'), self.text)
        st.add_assignment(Column(db_field='text_set'), set(("foo", "bar")))
        st.add_assignment(Column(db_field='text_list'), ["foo", "bar"])
        st.add_assignment(Column(db_field='text_map'), {"foo": '1', "bar": '2'})

        execute(st)
        self._verify_statement(st)
Exemplo n.º 21
0
def create_keyspace(name, strategy_class, replication_factor, durable_writes=True, **replication_values):
    """
    *Deprecated - use :func:`create_keyspace_simple` or :func:`create_keyspace_network_topology` instead*

    Creates a keyspace

    If the keyspace already exists, it will not be modified.

    **This function should be used with caution, especially in production environments.
    Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).**

    *There are plans to guard schema-modifying functions with an environment-driven conditional.*

    :param str name: name of keyspace to create
    :param str strategy_class: keyspace replication strategy class (:attr:`~.SimpleStrategy` or :attr:`~.NetworkTopologyStrategy`
    :param int replication_factor: keyspace replication factor, used with :attr:`~.SimpleStrategy`
    :param bool durable_writes: Write log is bypassed if set to False
    :param \*\*replication_values: Additional values to ad to the replication options map
    """
    if not _allow_schema_modification():
        return

    msg = "Deprecated. Use create_keyspace_simple or create_keyspace_network_topology instead"
    warnings.warn(msg, DeprecationWarning)
    log.warning(msg)

    cluster = get_cluster()

    if name not in cluster.metadata.keyspaces:
        # try the 1.2 method
        replication_map = {
            'class': strategy_class,
            'replication_factor': replication_factor
        }
        replication_map.update(replication_values)
        if strategy_class.lower() != 'simplestrategy':
            # Although the Cassandra documentation states for `replication_factor`
            # that it is "Required if class is SimpleStrategy; otherwise,
            # not used." we get an error if it is present.
            replication_map.pop('replication_factor', None)

        query = """
        CREATE KEYSPACE {}
        WITH REPLICATION = {}
        """.format(metadata.protect_name(name), json.dumps(replication_map).replace('"', "'"))

        if strategy_class != 'SimpleStrategy':
            query += " AND DURABLE_WRITES = {}".format('true' if durable_writes else 'false')

        execute(query)
Exemplo n.º 22
0
def update_compaction(model):
    """Updates the compaction options for the given model if necessary.

    :param model: The model to update.

    :return: `True`, if the compaction options were modified in Cassandra,
        `False` otherwise.
    :rtype: bool
    """
    log.debug("Checking %s for compaction differences", model)
    table = get_table_settings(model)

    existing_options = table.options.copy()

    existing_compaction_strategy = existing_options[
        'compaction_strategy_class']

    existing_options = json.loads(
        existing_options['compaction_strategy_options'])

    desired_options = get_compaction_options(model)

    desired_compact_strategy = desired_options.get(
        'class', SizeTieredCompactionStrategy)

    desired_options.pop('class', None)

    do_update = False

    if desired_compact_strategy not in existing_compaction_strategy:
        do_update = True

    for k, v in desired_options.items():
        val = existing_options.pop(k, None)
        if val != v:
            do_update = True

    # check compaction_strategy_options
    if do_update:
        options = get_compaction_options(model)
        # jsonify
        options = json.dumps(options).replace('"', "'")
        cf_name = model.column_family_name()
        query = "ALTER TABLE {0} with compaction = {1}".format(
            cf_name, options)
        execute(query)
        return True

    return False
Exemplo n.º 23
0
def _drop_table(model):
    if not _allow_schema_modification():
        return

    # don't try to delete non existant tables
    meta = get_cluster().metadata

    ks_name = model._get_keyspace()
    raw_cf_name = model._raw_column_family_name()

    try:
        meta.keyspaces[ks_name].tables[raw_cf_name]
        execute('DROP TABLE {0};'.format(model.column_family_name()))
    except KeyError:
        pass
Exemplo n.º 24
0
def _drop_table(model):
    if not _allow_schema_modification():
        return

    # don't try to delete non existant tables
    meta = get_cluster().metadata

    ks_name = model._get_keyspace()
    raw_cf_name = model._raw_column_family_name()

    try:
        meta.keyspaces[ks_name].tables[raw_cf_name]
        execute('DROP TABLE {0};'.format(model.column_family_name()))
    except KeyError:
        pass
Exemplo n.º 25
0
def _update_options(model, connection=None):
    """Updates the table options for the given model if necessary.

    :param model: The model to update.
    :param connection: Name of the connection to use

    :return: `True`, if the options were modified in Cassandra,
        `False` otherwise.
    :rtype: bool
    """
    ks_name = model._get_keyspace()
    msg = format_log_context("Checking %s for option differences", keyspace=ks_name, connection=connection)
    log.debug(msg, model)
    model_options = model.__options__ or {}

    table_meta = _get_table_metadata(model, connection=connection)
    # go to CQL string first to normalize meta from different versions
    existing_option_strings = set(table_meta._make_option_strings(table_meta.options))
    existing_options = _options_map_from_strings(existing_option_strings)
    model_option_strings = metadata.TableMetadataV3._make_option_strings(model_options)
    model_options = _options_map_from_strings(model_option_strings)

    update_options = {}
    for name, value in model_options.items():
        try:
            existing_value = existing_options[name]
        except KeyError:
            msg = format_log_context("Invalid table option: '%s'; known options: %s", keyspace=ks_name, connection=connection)
            raise KeyError(msg % (name, existing_options.keys()))
        if isinstance(existing_value, six.string_types):
            if value != existing_value:
                update_options[name] = value
        else:
            try:
                for k, v in value.items():
                    if existing_value[k] != v:
                        update_options[name] = value
                        break
            except KeyError:
                update_options[name] = value

    if update_options:
        options = ' AND '.join(metadata.TableMetadataV3._make_option_strings(update_options))
        query = "ALTER TABLE {0} WITH {1}".format(model.column_family_name(), options)
        execute(query, connection=connection)
        return True

    return False
Exemplo n.º 26
0
def search_data_cassandra(hit, admin_code):
    """[summary]

    Args:
        hit ([type]): [description]
        admin_code ([type]): [description]

    Returns:
        [type]: [description]
    """
    #recherche des données de la liste des geonameid liée à la cible dans Cassandra

    list_geonameid = search_zone(admin_code, hit, "geonames")
    #recherche des geonames selon la zone indiquée
    list_geonameid_hierarchy = search_zone("admin4_code", hit, "geonames")
    #recuperation des geoname ayant le même CP car tous les geonames
    #ne sont pas associés à la table Hierarchie
    hierarchy = search_data_hierarchy_fr(list_geonameid_hierarchy)

    if list_geonameid == -1:
        return "L'admin_code n'existe pas"

    list_geonameid = list_to_str(list_geonameid)

    list_geoname = []
    query = 'SELECT * FROM cassandra.geo_name WHERE geonameid IN (' + list_geonameid + ')'
    rows = connection.execute(query)

    for row in rows:
        list_geoname.append(row)

    return list_geoname, hierarchy
Exemplo n.º 27
0
def fill(entity_name, base, batch_size, mongo_ids=[], neo_ids=[], strings=[]):
    data = json.load(
        open(
            convert(
                os.environ['SPACE_SHIP_HOME'] +
                '/generation/dummyMarket/cassandra json/{0}.json'.format(base),
                base,
                mongo_ids=mongo_ids,
                neo_ids=neo_ids,
                strings=strings)))

    number_of_batches = (len(data["data"]) // batch_size) + 1

    for i in range(number_of_batches + 1):
        if i < number_of_batches:
            query = "BEGIN BATCH {0} APPLY BATCH;".format(' '.join([
                "insert into {0}.{1} json '{2}';".format(
                    DB_NAME, entity_name, json.dumps(item))
                for item in data["data"][i * batch_size:(i + 1) * batch_size]
            ]))
        else:
            query = "BEGIN BATCH {0} APPLY BATCH;".format(' '.join([
                "insert into {0}.{1} json '{2}';".format(
                    DB_NAME, entity_name, json.dumps(item))
                for item in data["data"][i * batch_size:]
            ]))
        print(connection.execute(query))
Exemplo n.º 28
0
def update(table, params, dicted=False):
    where = {}
    update = {}
    for param in params:
        if 'set_' in param:
            update[param.replace('set_', '')] = params[param]
        else:
            where[param] = params[param]

    select_result = select(table, where, dicted=True)
    parsed_update_params = parse_params(update, ', ')

    #result = connection.execute
    #print('BEGIN BATCH ' + ' '.join(['update {0}.{1} set {3} where {2};'.format(DB_NAME, table,
    #	extract_keys(item, ['date', 'time']), parsed_update_params) for item in select_result ]) + ' APPLY BATCH;')

    result = connection.execute('BEGIN BATCH ' + ' '.join([
        'update {0}.{1} set {3} where {2};'.format(
            DB_NAME, table, extract_keys(item, ['date', 'time']),
            parsed_update_params) for item in select_result
    ]) + ' APPLY BATCH;').current_rows
    #.current_rows
    if not dicted:
        return [
            namedtuple('Struct', item.keys())(*item.values())
            for item in result
        ]
    return result
Exemplo n.º 29
0
def update(table_name, columns_getter, columns_setter):
	columns_filter = ', '.join([item[0] for item in columns_getter])
	
	values_filter = ' and '.join([\
		item[0] + ' = ' + str(item[1])\
		if isinstance(item[1], int) or isinstance(item[1], float) else\
		item[0] + ' = \'' + str(item[1]) + '\'' \
		for item in (item for item in columns_getter if len(item) > 1)\
	])

	for column in columns_setter:
		if len(column) > 1:
			try:
				column[1] = int(column[1])
			except ValueError:
				try:
					column[1] = float(column[1])
				except ValueError:
					pass

	values_setter = ', '.join([\
		item[0] + ' = ' + str(item[1])\
		if isinstance(item[1], int) or isinstance(item[1], float) else\
		item[0] + ' = \'' + str(item[1]) + '\'' \
		for item in (item for item in columns_setter if len(item) > 1)\
	])

	if len(values_filter) > 0:
		values_filter = 'where ' + values_filter
	
	if len(values_setter) > 0:
		values_setter = 'set ' + values_setter

	print('update {0}.{1} {2} {3} allow filtering;'.format(DB_NAME, table_name, values_filter, values_setter))
	return connection.execute('update {0}.{1} {3} {2};'.format(DB_NAME, table_name, values_filter, values_setter)).current_rows
Exemplo n.º 30
0
 def _execute(self, q):
     if self._batch:
         return self._batch.add_query(q)
     else:
         tmp = connection.execute(q, consistency_level=self._consistency, timeout=self._timeout)
         if self._if_not_exists or self._if_exists or self._conditional:
             check_applied(tmp)
         return tmp
Exemplo n.º 31
0
    def _verify_statement(self, original):
        st = SelectStatement(self.table_name)
        result = execute(st)
        response = result[0]

        for assignment in original.assignments:
            self.assertEqual(response[assignment.field], assignment.value)
        self.assertEqual(len(response), 7)
Exemplo n.º 32
0
 def _execute(self, q):
     if self._batch:
         return self._batch.add_query(q)
     else:
         result = connection.execute(q, consistency_level=self._consistency, timeout=self._timeout)
         if self._if_not_exists or self._if_exists or self._transaction:
             check_applied(result)
         return result
Exemplo n.º 33
0
 def _execute(self, q):
     if self._batch:
         return self._batch.add_query(q)
     else:
         tmp = connection.execute(q, consistency_level=self._consistency, timeout=self._timeout)
         if self._if_not_exists or self._transaction:
             check_applied(tmp)
         return tmp
Exemplo n.º 34
0
    def _verify_statement(self, original):
        st = SelectStatement(self.table_name)
        result = execute(st)
        response = result[0]

        for assignment in original.assignments:
            self.assertEqual(response[assignment.field], assignment.value)
        self.assertEqual(len(response), 7)
Exemplo n.º 35
0
def drop_keyspace(name):
    """
    Drops a keyspace, if it exists.

    *There are plans to guard schema-modifying functions with an environment-driven conditional.*

    **This function should be used with caution, especially in production environments.
    Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).**

    :param str name: name of keyspace to drop
    """
    if not _allow_schema_modification():
        return

    cluster = get_cluster()
    if name in cluster.metadata.keyspaces:
        execute("DROP KEYSPACE {}".format(metadata.protect_name(name)))
Exemplo n.º 36
0
def drop_keyspace(name):
    """
    Drops a keyspace, if it exists.

    *There are plans to guard schema-modifying functions with an environment-driven conditional.*

    **This function should be used with caution, especially in production environments.
    Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).**

    :param str name: name of keyspace to drop
    """
    if not _allow_schema_modification():
        return

    cluster = get_cluster()
    if name in cluster.metadata.keyspaces:
        execute("DROP KEYSPACE {0}".format(metadata.protect_name(name)))
Exemplo n.º 37
0
 def _execute(self, q):
     if self._batch:
         return self._batch.add_query(q)
     else:
         result = connection.execute(q, consistency_level=self._consistency, timeout=self._timeout)
         if self._transaction:
             check_applied(result)
         return result
Exemplo n.º 38
0
def _sync_type(ks_name, type_model, omit_subtypes=None):

    syncd_sub_types = omit_subtypes or set()
    for field in type_model._fields.values():
        udts = []
        columns.resolve_udts(field, udts)
        for udt in [u for u in udts if u not in syncd_sub_types]:
            _sync_type(ks_name, udt, syncd_sub_types)
            syncd_sub_types.add(udt)

    type_name = type_model.type_name()
    type_name_qualified = "%s.%s" % (ks_name, type_name)

    cluster = get_cluster()

    keyspace = cluster.metadata.keyspaces[ks_name]
    defined_types = keyspace.user_types

    if type_name not in defined_types:
        log.debug("sync_type creating new type %s", type_name_qualified)
        cql = get_create_type(type_model, ks_name)
        execute(cql)
        cluster.refresh_user_type_metadata(ks_name, type_name)
        type_model.register_for_keyspace(ks_name)
    else:
        defined_fields = defined_types[type_name].field_names
        model_fields = set()
        for field in type_model._fields.values():
            model_fields.add(field.db_field_name)
            if field.db_field_name not in defined_fields:
                execute("ALTER TYPE {0} ADD {1}".format(
                    type_name_qualified, field.get_column_def()))

        type_model.register_for_keyspace(ks_name)

        if len(defined_fields) == len(model_fields):
            log.info("Type %s did not require synchronization",
                     type_name_qualified)
            return

        db_fields_not_in_model = model_fields.symmetric_difference(
            defined_fields)
        if db_fields_not_in_model:
            log.info("Type %s has fields not referenced by model: %s",
                     type_name_qualified, db_fields_not_in_model)
Exemplo n.º 39
0
def update_compaction(model):
    """Updates the compaction options for the given model if necessary.

    :param model: The model to update.

    :return: `True`, if the compaction options were modified in Cassandra,
        `False` otherwise.
    :rtype: bool
    """
    log.debug("Checking %s for compaction differences", model)
    table = get_table_settings(model)

    existing_options = table.options.copy()

    existing_compaction_strategy = existing_options['compaction_strategy_class']

    existing_options = json.loads(existing_options['compaction_strategy_options'])

    desired_options = get_compaction_options(model)

    desired_compact_strategy = desired_options.get('class', SizeTieredCompactionStrategy)

    desired_options.pop('class', None)

    do_update = False

    if desired_compact_strategy not in existing_compaction_strategy:
        do_update = True

    for k, v in desired_options.items():
        val = existing_options.pop(k, None)
        if val != v:
            do_update = True

    # check compaction_strategy_options
    if do_update:
        options = get_compaction_options(model)
        # jsonify
        options = json.dumps(options).replace('"', "'")
        cf_name = model.column_family_name()
        query = "ALTER TABLE {} with compaction = {}".format(cf_name, options)
        execute(query)
        return True

    return False
Exemplo n.º 40
0
def _execute_statement(model, statement, consistency_level, timeout):
    params = statement.get_context()
    s = SimpleStatement(str(statement), consistency_level=consistency_level, fetch_size=statement.fetch_size)
    if model._partition_key_index:
        key_values = statement.partition_key_values(model._partition_key_index)
        if not any(v is None for v in key_values):
            parts = model._routing_key_from_values(key_values, connection.get_cluster().protocol_version)
            s.routing_key = parts
            s.keyspace = model._get_keyspace()
    return connection.execute(s, params, timeout=timeout)
Exemplo n.º 41
0
def _execute_statement(model, statement, consistency_level, timeout):
    params = statement.get_context()
    s = SimpleStatement(str(statement), consistency_level=consistency_level, fetch_size=statement.fetch_size)
    if model._partition_key_index:
        key_values = statement.partition_key_values(model._partition_key_index)
        if not any(v is None for v in key_values):
            parts = model._routing_key_from_values(key_values, connection.get_cluster().protocol_version)
            s.routing_key = parts
            s.keyspace = model._get_keyspace()
    return connection.execute(s, params, timeout=timeout)
Exemplo n.º 42
0
def _update_options(model):
    """Updates the table options for the given model if necessary.

    :param model: The model to update.

    :return: `True`, if the options were modified in Cassandra,
        `False` otherwise.
    :rtype: bool
    """
    log.debug("Checking %s for option differences", model)
    model_options = model.__options__ or {}

    table_meta = _get_table_metadata(model)
    # go to CQL string first to normalize meta from different versions
    existing_option_strings = set(table_meta._make_option_strings(table_meta.options))
    existing_options = _options_map_from_strings(existing_option_strings)
    model_option_strings = metadata.TableMetadataV3._make_option_strings(model_options)
    model_options = _options_map_from_strings(model_option_strings)

    update_options = {}
    for name, value in model_options.items():
        existing_value = existing_options[name]
        if isinstance(existing_value, six.string_types):
            if value != existing_value:
                update_options[name] = value
        else:
            try:
                for k, v in value.items():
                    if existing_value[k] != v:
                        update_options[name] = value
                        break
            except KeyError:
                update_options[name] = value

    if update_options:
        options = ' AND '.join(metadata.TableMetadataV3._make_option_strings(update_options))
        query = "ALTER TABLE {0} WITH {1}".format(model.column_family_name(), options)
        execute(query)
        return True

    return False
Exemplo n.º 43
0
def _sync_type(ks_name, type_model, omit_subtypes=None):

    syncd_sub_types = omit_subtypes or set()
    for field in type_model._fields.values():
        udts = []
        columns.resolve_udts(field, udts)
        for udt in [u for u in udts if u not in syncd_sub_types]:
            _sync_type(ks_name, udt, syncd_sub_types)
            syncd_sub_types.add(udt)

    type_name = type_model.type_name()
    type_name_qualified = "%s.%s" % (ks_name, type_name)

    cluster = get_cluster()

    keyspace = cluster.metadata.keyspaces[ks_name]
    defined_types = keyspace.user_types

    if type_name not in defined_types:
        log.debug("sync_type creating new type %s", type_name_qualified)
        cql = get_create_type(type_model, ks_name)
        execute(cql)
        cluster.refresh_user_type_metadata(ks_name, type_name)
        type_model.register_for_keyspace(ks_name)
    else:
        defined_fields = defined_types[type_name].field_names
        model_fields = set()
        for field in type_model._fields.values():
            model_fields.add(field.db_field_name)
            if field.db_field_name not in defined_fields:
                execute("ALTER TYPE {} ADD {}".format(type_name_qualified, field.get_column_def()))

        type_model.register_for_keyspace(ks_name)

        if len(defined_fields) == len(model_fields):
            log.info("Type %s did not require synchronization", type_name_qualified)
            return

        db_fields_not_in_model = model_fields.symmetric_difference(defined_fields)
        if db_fields_not_in_model:
            log.info("Type %s has fields not referenced by model: %s", type_name_qualified, db_fields_not_in_model)
Exemplo n.º 44
0
def sync_materialized_view(mv):
    viewName = getattr(mv, 'view_name', mv.__name__)
    partitionKeys = mv.partition_keys
    primaryKeys = getattr(mv, 'primary_keys', None)
    basePrimaryKeys = getattr(mv, 'base_primary_keys', ['id'])
    baseTableName = getattr(mv, 'base_table_name', None)
    if not baseTableName:
        baseTableName = viewName.split('_by_')[0]
    cols = mv._defined_columns  # key is col name
    select = ','.join(colName for colName in cols)
    where = ' AND '.join('%s IS NOT NULL' % (key) for key in partitionKeys)
    primary = ['(%s)' % (','.join(partitionKeys))]
    if primaryKeys:
        primary.append(','.join(primaryKeys))
    primary.append(','.join(basePrimaryKeys))
    from cassandra.cqlengine import models
    keyspace = models.DEFAULT_KEYSPACE
    connection.execute('use %s;' % (keyspace))
    connection.execute('DROP MATERIALIZED VIEW IF EXISTS %s;' % (viewName))
    cql = """
CREATE MATERIALIZED VIEW %s AS
  SELECT %s FROM %s
  WHERE %s
PRIMARY KEY (%s);
""" % (viewName, select, baseTableName, where, ','.join(primary))
    connection.execute(cql)
Exemplo n.º 45
0
def select(table_name, columns):
	columns_filter = ', '.join([item[0] for item in columns])
	values_filter = ' and '.join([\
		item[0] + ' = ' + str(item[1])\
		if isinstance(item[1], int) or isinstance(item[1], float) else\
		item[0] + ' = \'' + str(item[1]) + '\'' \
		for item in (item for item in columns if len(item) > 1)\
	])

	if len(values_filter) > 0:
		values_filter = 'where ' + values_filter
	print('select {2} from {0}.{1} {3} allow filtering;'.format(DB_NAME, table_name, columns_filter, values_filter))
	return connection.execute('select {2} from {0}.{1} {3} allow filtering;'.format(DB_NAME, table_name, columns_filter, values_filter)).current_rows
Exemplo n.º 46
0
def select(table, params, dicted=False):
    #print('select * from {0}.{1} where {2} allow filtering;'.format(DB_NAME, table, parse_params(params, ' and ')))
    parsed_params = parse_params(params, ' and ')
    result = connection.execute('select * from {0}.{1} {2};'.format(
        DB_NAME, table, 'where {0} allow filtering'.format(parsed_params)
        if len(parsed_params) else '')).current_rows

    if not dicted:
        return [
            namedtuple('Struct', item.keys())(*item.values())
            for item in result
        ]
    return result
Exemplo n.º 47
0
def fill(entity_name, base, batch_size, mongo_ids = [], neo_ids = [], strings = []):
	data = json.load(open(convert(os.environ['SPACE_SHIP_HOME'] + '/generation/dummyMarket/cassandra json/{0}.json'.format(base), base, mongo_ids = mongo_ids,
		neo_ids = neo_ids, strings = strings)))

	number_of_batches = (len(data["data"]) // batch_size) + 1

	for i in range(number_of_batches + 1):
		if i < number_of_batches:	
			query = "BEGIN BATCH {0} APPLY BATCH;".format(' '.join(
				["insert into {0}.{1} json '{2}';".format(DB_NAME, entity_name, json.dumps(item)) for item in data["data"][i*batch_size : (i + 1)*batch_size]]))
		else:
			query = "BEGIN BATCH {0} APPLY BATCH;".format(' '.join(
				["insert into {0}.{1} json '{2}';".format(DB_NAME, entity_name, json.dumps(item)) for item in data["data"][i*batch_size :]]))
		print(connection.execute(query))
Exemplo n.º 48
0
def drop_table(model):
    """
    Drops the table indicated by the model, if it exists.

    **This function should be used with caution, especially in production environments.
    Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).**

    *There are plans to guard schema-modifying functions with an environment-driven conditional.*
    """
    if not _allow_schema_modification():
        return

    # don't try to delete non existant tables
    meta = get_cluster().metadata

    ks_name = model._get_keyspace()
    raw_cf_name = model._raw_column_family_name()

    try:
        meta.keyspaces[ks_name].tables[raw_cf_name]
        execute('DROP TABLE {0};'.format(model.column_family_name()))
    except KeyError:
        pass
Exemplo n.º 49
0
def drop_table(model):
    """
    Drops the table indicated by the model, if it exists.

    **This function should be used with caution, especially in production environments.
    Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).**

    *There are plans to guard schema-modifying functions with an environment-driven conditional.*
    """
    if not _allow_schema_modification():
        return

    # don't try to delete non existant tables
    meta = get_cluster().metadata

    ks_name = model._get_keyspace()
    raw_cf_name = model._raw_column_family_name()

    try:
        meta.keyspaces[ks_name].tables[raw_cf_name]
        execute('DROP TABLE {};'.format(model.column_family_name()))
    except KeyError:
        pass
Exemplo n.º 50
0
def get_fields(model):
    # returns all fields that aren't part of the PK
    ks_name = model._get_keyspace()
    col_family = model._raw_column_family_name()
    field_types = ['regular', 'static']
    query = "select * from system.schema_columns where keyspace_name = %s and columnfamily_name = %s"
    tmp = execute(query, [ks_name, col_family])

    # Tables containing only primary keys do not appear to create
    # any entries in system.schema_columns, as only non-primary-key attributes
    # appear to be inserted into the schema_columns table
    try:
        return [Field(x['column_name'], x['validator']) for x in tmp if x['type'] in field_types]
    except KeyError:
        return [Field(x['column_name'], x['validator']) for x in tmp]
Exemplo n.º 51
0
def select(table_name, columns):
    columns_filter = ', '.join([item[0] for item in columns])
    values_filter = ' and '.join([\
     item[0] + ' = ' + str(item[1])\
     if isinstance(item[1], int) or isinstance(item[1], float) else\
     item[0] + ' = \'' + str(item[1]) + '\'' \
     for item in (item for item in columns if len(item) > 1)\
    ])

    if len(values_filter) > 0:
        values_filter = 'where ' + values_filter
    print('select {2} from {0}.{1} {3} allow filtering;'.format(
        DB_NAME, table_name, columns_filter, values_filter))
    return connection.execute(
        'select {2} from {0}.{1} {3} allow filtering;'.format(
            DB_NAME, table_name, columns_filter, values_filter)).current_rows
Exemplo n.º 52
0
    def execute(self):
        if self._executed and self.warn_multiple_exec:
            msg = "Batch executed multiple times."
            if self._context_entered:
                msg += " If using the batch as a context manager, there is no need to call execute directly."
            warn(msg)
        self._executed = True

        if len(self.queries) == 0:
            # Empty batch is a no-op
            # except for callbacks
            self._execute_callbacks()
            return

        opener = 'BEGIN ' + (self.batch_type + ' ' if self.batch_type else '') + ' BATCH'
        if self.timestamp:

            if isinstance(self.timestamp, six.integer_types):
                ts = self.timestamp
            elif isinstance(self.timestamp, (datetime, timedelta)):
                ts = self.timestamp
                if isinstance(self.timestamp, timedelta):
                    ts += datetime.now()  # Apply timedelta
                ts = int(time.mktime(ts.timetuple()) * 1e+6 + ts.microsecond)
            else:
                raise ValueError("Batch expects a long, a timedelta, or a datetime")

            opener += ' USING TIMESTAMP {0}'.format(ts)

        query_list = [opener]
        parameters = {}
        ctx_counter = 0
        for query in self.queries:
            query.update_context_id(ctx_counter)
            ctx = query.get_context()
            ctx_counter += len(ctx)
            query_list.append('  ' + str(query))
            parameters.update(ctx)

        query_list.append('APPLY BATCH;')

        tmp = connection.execute('\n'.join(query_list), parameters, self._consistency, self._timeout)
        check_applied(tmp)

        self.queries = []
        self._execute_callbacks()
Exemplo n.º 53
0
    def execute(self):
        if self._executed and self.warn_multiple_exec:
            msg = "Batch executed multiple times."
            if self._context_entered:
                msg += " If using the batch as a context manager, there is no need to call execute directly."
            warn(msg)
        self._executed = True

        if len(self.queries) == 0:
            # Empty batch is a no-op
            # except for callbacks
            self._execute_callbacks()
            return

        opener = 'BEGIN ' + (self.batch_type + ' ' if self.batch_type else '') + ' BATCH'
        if self.timestamp:

            if isinstance(self.timestamp, six.integer_types):
                ts = self.timestamp
            elif isinstance(self.timestamp, (datetime, timedelta)):
                ts = self.timestamp
                if isinstance(self.timestamp, timedelta):
                    ts += datetime.now()  # Apply timedelta
                ts = int(time.mktime(ts.timetuple()) * 1e+6 + ts.microsecond)
            else:
                raise ValueError("Batch expects a long, a timedelta, or a datetime")

            opener += ' USING TIMESTAMP {0}'.format(ts)

        query_list = [opener]
        parameters = {}
        ctx_counter = 0
        for query in self.queries:
            query.update_context_id(ctx_counter)
            ctx = query.get_context()
            ctx_counter += len(ctx)
            query_list.append('  ' + str(query))
            parameters.update(ctx)

        query_list.append('APPLY BATCH;')

        tmp = connection.execute('\n'.join(query_list), parameters, self._consistency, self._timeout)
        check_applied(tmp)

        self.queries = []
        self._execute_callbacks()
Exemplo n.º 54
0
def get_fields(model):
    # returns all fields that aren't part of the PK
    ks_name = model._get_keyspace()
    col_family = model.column_family_name(include_keyspace=False)
    field_types = ['regular', 'static']
    query = "select * from system.schema_columns where keyspace_name = %s and columnfamily_name = %s"
    tmp = execute(query, [ks_name, col_family])

    # Tables containing only primary keys do not appear to create
    # any entries in system.schema_columns, as only non-primary-key attributes
    # appear to be inserted into the schema_columns table
    try:
        return [
            Field(x['column_name'], x['validator']) for x in tmp
            if x['type'] in field_types
        ]
    except KeyError:
        return [Field(x['column_name'], x['validator']) for x in tmp]
Exemplo n.º 55
0
    def execute(self):
        if len(self.queries) == 0:
            # Empty batch is a no-op
            # except for callbacks
            self._execute_callbacks()
            return

        opener = "BEGIN " + (self.batch_type + " " if self.batch_type else "") + " BATCH"
        if self.timestamp:

            if isinstance(self.timestamp, six.integer_types):
                ts = self.timestamp
            elif isinstance(self.timestamp, (datetime, timedelta)):
                ts = self.timestamp
                if isinstance(self.timestamp, timedelta):
                    ts += datetime.now()  # Apply timedelta
                ts = int(time.mktime(ts.timetuple()) * 1e6 + ts.microsecond)
            else:
                raise ValueError("Batch expects a long, a timedelta, or a datetime")

            opener += " USING TIMESTAMP {0}".format(ts)

        query_list = [opener]
        parameters = {}
        ctx_counter = 0
        for query in self.queries:
            query.update_context_id(ctx_counter)
            ctx = query.get_context()
            ctx_counter += len(ctx)
            query_list.append("  " + str(query))
            parameters.update(ctx)

        query_list.append("APPLY BATCH;")

        tmp = connection.execute("\n".join(query_list), parameters, self._consistency, self._timeout)
        check_applied(tmp)

        self.queries = []
        self._execute_callbacks()
Exemplo n.º 56
0
    def test_like_operator(self):
        """
        Test to verify the like operator works appropriately

        @since 3.13
        @jira_ticket PYTHON-512
        @expected_result the expected row is read using LIKE

        @test_category data_types:object_mapper
        """
        cluster = Cluster()
        session = cluster.connect()
        self.addCleanup(cluster.shutdown)

        session.execute("""CREATE CUSTOM INDEX text_index ON {} (text)
                                    USING 'org.apache.cassandra.index.sasi.SASIIndex';""".format(self.table_name))
        self.addCleanup(session.execute, "DROP INDEX {}.text_index".format(DEFAULT_KEYSPACE))

        partition = uuid4()
        cluster = 1
        self._insert_statement(partition, cluster)

        ss = SelectStatement(self.table_name)
        like_clause = "text_for_%"
        ss.add_where(Column(db_field='text'), LikeOperator(), like_clause)
        self.assertEqual(six.text_type(ss),
                         'SELECT * FROM {} WHERE "text" LIKE %(0)s'.format(self.table_name))

        result = execute(ss)
        self.assertEqual(result[0]["text"], self.text)

        q = TestQueryUpdateModel.objects.filter(text__like=like_clause).allow_filtering()
        self.assertEqual(q[0].text, self.text)

        q = TestQueryUpdateModel.objects.filter(text__like=like_clause)
        self.assertEqual(q[0].text, self.text)
Exemplo n.º 57
0
def sync_table(model):
    """
    Inspects the model and creates / updates the corresponding table and columns.

    Any User Defined Types used in the table are implicitly synchronized.

    This function can only add fields that are not part of the primary key.

    Note that the attributes removed from the model are not deleted on the database.
    They become effectively ignored by (will not show up on) the model.

    **This function should be used with caution, especially in production environments.
    Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).**

    *There are plans to guard schema-modifying functions with an environment-driven conditional.*
    """
    if not _allow_schema_modification():
        return

    if not issubclass(model, Model):
        raise CQLEngineException("Models must be derived from base Model.")

    if model.__abstract__:
        raise CQLEngineException("cannot create table from abstract model")

    cf_name = model.column_family_name()
    raw_cf_name = model._raw_column_family_name()

    ks_name = model._get_keyspace()

    cluster = get_cluster()

    keyspace = cluster.metadata.keyspaces[ks_name]
    tables = keyspace.tables

    syncd_types = set()
    for col in model._columns.values():
        udts = []
        columns.resolve_udts(col, udts)
        for udt in [u for u in udts if u not in syncd_types]:
            _sync_type(ks_name, udt, syncd_types)

    # check for an existing column family
    if raw_cf_name not in tables:
        log.debug("sync_table creating new table %s", cf_name)
        qs = get_create_table(model)

        try:
            execute(qs)
        except CQLEngineException as ex:
            # 1.2 doesn't return cf names, so we have to examine the exception
            # and ignore if it says the column family already exists
            if "Cannot add already existing column family" not in unicode(ex):
                raise
    else:
        log.debug("sync_table checking existing table %s", cf_name)
        # see if we're missing any columns
        fields = get_fields(model)
        field_names = [x.name for x in fields]
        model_fields = set()
        # # TODO: does this work with db_name??
        for name, col in model._columns.items():
            if col.primary_key or col.partition_key:
                continue  # we can't mess with the PK
            model_fields.add(name)
            if col.db_field_name in field_names:
                continue  # skip columns already defined

            # add missing column using the column def
            query = "ALTER TABLE {} add {}".format(cf_name, col.get_column_def())
            execute(query)

        db_fields_not_in_model = model_fields.symmetric_difference(field_names)
        if db_fields_not_in_model:
            log.info("Table %s has fields not referenced by model: %s", cf_name, db_fields_not_in_model)

        update_compaction(model)

    table = cluster.metadata.keyspaces[ks_name].tables[raw_cf_name]

    indexes = [c for n, c in model._columns.items() if c.index]

    for column in indexes:
        if table.columns[column.db_field_name].index:
            continue

        qs = ['CREATE INDEX index_{}_{}'.format(raw_cf_name, column.db_field_name)]
        qs += ['ON {}'.format(cf_name)]
        qs += ['("{}")'.format(column.db_field_name)]
        qs = ' '.join(qs)
        execute(qs)
Exemplo n.º 58
0
def test_keyspace(verbose=False):
    connection.execute("USE {};".format(settings.CASSANDRA_CONNECTION['KEYSPACE']))
    if verbose:
        print("--- Keyspace OK")