def create_schema(self, model, waiting_models): """ Creates search schemas. Args: model: model to execute waiting_models: if riak can't return response immediately, model is taken to queue. After first execution session, method is executed with waiting models and controlled. And be ensured that all given models are executed properly. Returns: """ bucket_name = model._get_bucket_name() index_name = "%s_%s" % (settings.DEFAULT_BUCKET_TYPE, bucket_name) ins = model(fake_context) fields = self.get_schema_fields(ins._collect_index_fields()) new_schema = self.compile_schema(fields) schema = get_schema_from_solr(index_name) if not (schema == new_schema): try: client.create_search_schema(index_name, new_schema) print("+ %s (%s) search schema is created." % (model.__name__, index_name)) except: print( "+ %s (%s) search schema checking operation is taken to queue." % (model.__name__, index_name)) waiting_models.append(model)
def apply_schema(client, new_schema, model, silent=False): """ riak doesn't support schema/index updates ( http://git.io/vLOTS ) as a workaround, we create a temporary index, attach it to the bucket, delete the old index/schema, re-create the index with new schema, assign it to bucket, then delete the temporary index. :param byte new_schema: compiled schema :param str bucket_name: name of schema, index and bucket. :return: True or False :rtype: bool """ bucket_name = model._get_bucket_name() bucket_type = client.bucket_type(settings.DEFAULT_BUCKET_TYPE) bucket = bucket_type.bucket(bucket_name) n_val = bucket_type.get_property('n_val') # delete stale indexes # inuse_indexes = [b.get_properties().get('search_index') for b in # bucket_type.get_buckets()] # stale_indexes = [si['name'] for si in self.client.list_search_indexes() # if si['name'] not in inuse_indexes] # for stale_index in stale_indexes: # self.client.delete_search_index(stale_index) suffix = 9000000000 - int(time.time()) new_index_name = "%s_%s_%s" % (settings.DEFAULT_BUCKET_TYPE, bucket_name, suffix) client.create_search_schema(new_index_name, new_schema) client.create_search_index(new_index_name, new_index_name, n_val) bucket.set_property('search_index', new_index_name) # settings.update_index(bucket_name, new_index_name) if not silent: print("+ %s (%s)" % (model.__name__, new_index_name))
def apply_schema(client, force, job_pack, check_only): """ riak doesn't support schema/index updates ( http://git.io/vLOTS ) as a workaround, we create a temporary index, attach it to the bucket, delete the old index/schema, re-create the index with new schema, assign it to bucket, then delete the temporary index. :param byte new_schema: compiled schema :param str bucket_name: name of schema, index and bucket. :return: True or False :rtype: bool """ for new_schema, model in job_pack: try: bucket_name = model._get_bucket_name() bucket_type = client.bucket_type(settings.DEFAULT_BUCKET_TYPE) bucket = bucket_type.bucket(bucket_name) n_val = bucket_type.get_property('n_val') index_name = "%s_%s" % (settings.DEFAULT_BUCKET_TYPE, bucket_name) if not force: try: schema = get_schema_from_solr(index_name) if schema == new_schema: print("Schema %s is already up to date, nothing to do!" % index_name) continue elif check_only and schema != new_schema: print("Schema %s is not up to date, migrate this model!" % index_name) continue except: import traceback traceback.print_exc() bucket.set_property('search_index', 'foo_index') try: client.delete_search_index(index_name) except RiakError as e: if 'notfound' != e.value: raise wait_for_schema_deletion(index_name) client.create_search_schema(index_name, new_schema) client.create_search_index(index_name, index_name, n_val) bucket.set_property('search_index', index_name) print("+ %s (%s)" % (model.__name__, index_name)) stream = bucket.stream_keys() i = 0 unsaved_keys = [] for key_list in stream: for key in key_list: i += 1 # time.sleep(0.4) try: obj = bucket.get(key) if obj.data: obj.store() except ConflictError: unsaved_keys.append(key) print("Error on save. Record in conflict: %s > %s" % (bucket_name, key)) except: unsaved_keys.append(key) print("Error on save! %s > %s" % (bucket_name, key)) import traceback traceback.print_exc() stream.close() print("Re-indexed %s records of %s" % (i, bucket_name)) if unsaved_keys: print("\nThese keys cannot be updated:\n\n", unsaved_keys) except: print("n_val: %s" % n_val) print("bucket_name: %s" % bucket_name) print("bucket_type: %s" % bucket_type) raise
def apply_schema(client, force, job_pack): """ riak doesn't support schema/index updates ( http://git.io/vLOTS ) as a workaround, we create a temporary index, attach it to the bucket, delete the old index/schema, re-create the index with new schema, assign it to bucket, then delete the temporary index. :param byte new_schema: compiled schema :param str bucket_name: name of schema, index and bucket. :return: True or False :rtype: bool """ for new_schema, model in job_pack: try: bucket_name = model._get_bucket_name() bucket_type = client.bucket_type(settings.DEFAULT_BUCKET_TYPE) bucket = bucket_type.bucket(bucket_name) n_val = bucket_type.get_property('n_val') index_name = "%s_%s" % (settings.DEFAULT_BUCKET_TYPE, bucket_name) if not force: try: if get_schema_from_solr(index_name) == new_schema: print( "Schema %s already up to date, nothing to do!" % index_name) continue except: import traceback traceback.print_exc() bucket.set_property('search_index', 'foo_index') try: client.delete_search_index(index_name) except RiakError as e: if 'notfound' != e.value: raise wait_for_schema_deletion(index_name) client.create_search_schema(index_name, new_schema) client.create_search_index(index_name, index_name, n_val) bucket.set_property('search_index', index_name) print("+ %s (%s)" % (model.__name__, index_name)) stream = bucket.stream_keys() i = 0 unsaved_keys = [] for key_list in stream: for key in key_list: i += 1 # time.sleep(0.4) try: obj = bucket.get(key) if obj.data: obj.store() except ConflictError: unsaved_keys.append(key) print( "Error on save. Record in conflict: %s > %s" % (bucket_name, key)) except: unsaved_keys.append(key) print("Error on save! %s > %s" % (bucket_name, key)) import traceback traceback.print_exc() stream.close() print("Re-indexed %s records of %s" % (i, bucket_name)) if unsaved_keys: print("\nThese keys cannot be updated:\n\n", unsaved_keys) except: print("n_val: %s" % n_val) print("bucket_name: %s" % bucket_name) print("bucket_type: %s" % bucket_type) raise