def apply_schema(client, new_schema, model, silent=False):
"""
riak doesn't support schema/index updates ( http://git.io/vLOTS )
as a workaround, we create a temporary index,
attach it to the bucket, delete the old index/schema,
re-create the index with new schema, assign it to bucket,
then delete the temporary index.
:param byte new_schema: compiled schema
:param str bucket_name: name of schema, index and bucket.
:return: True or False
:rtype: bool
"""
bucket_name = model._get_bucket_name()
bucket_type = client.bucket_type(settings.DEFAULT_BUCKET_TYPE)
bucket = bucket_type.bucket(bucket_name)
n_val = bucket_type.get_property('n_val')
# delete stale indexes
# inuse_indexes = [b.get_properties().get('search_index') for b in
# bucket_type.get_buckets()]
# stale_indexes = [si['name'] for si in self.client.list_search_indexes()
# if si['name'] not in inuse_indexes]
# for stale_index in stale_indexes:
# self.client.delete_search_index(stale_index)
suffix = 9000000000 - int(time.time())
new_index_name = "%s_%s_%s" % (settings.DEFAULT_BUCKET_TYPE, bucket_name, suffix)
client.create_search_schema(new_index_name, new_schema)
client.create_search_index(new_index_name, new_index_name, n_val)
bucket.set_property('search_index', new_index_name)
# settings.update_index(bucket_name, new_index_name)
if not silent:
print("+ %s (%s)" % (model.__name__, new_index_name))
def create_index(self, model, waiting_models):
"""
Creates search indexes.
Args:
model: model to execute
waiting_models: if riak can't return response immediately, model is taken to queue.
After first execution session, method is executed with waiting models and controlled.
And be ensured that all given models are executed properly.
Returns:
"""
bucket_name = model._get_bucket_name()
bucket_type = client.bucket_type(settings.DEFAULT_BUCKET_TYPE)
index_name = "%s_%s" % (settings.DEFAULT_BUCKET_TYPE, bucket_name)
bucket = bucket_type.bucket(bucket_name)
try:
client.get_search_index(index_name)
if not (bucket.get_property('search_index') == index_name):
bucket.set_property('search_index', index_name)
print("+ %s (%s) search index is created." %
(model.__name__, index_name))
except RiakError:
try:
client.create_search_index(index_name, index_name, self.n_val)
bucket.set_property('search_index', index_name)
print("+ %s (%s) search index is created." %
(model.__name__, index_name))
except RiakError:
print(
"+ %s (%s) search index checking operation is taken to queue."
% (model.__name__, index_name))
waiting_models.append(model)
def __init__(self, models, threads, force):
self.report = []
self.models = models
self.force = force
self.client = client
self.threads = int(threads)
self.n_val = client.bucket_type(
settings.DEFAULT_BUCKET_TYPE).get_property('n_val')
self.base_thread = BaseThreadedCommand()
def birim_kaydet(self, birim_id):
# self.kvdb.conn.set(birim_id, self.birim_detaylari())
# self.logger.info("%s icin degerler: %s\n\n" % (birim_id, self.birim_detaylari()))
from pyoko.db.connection import client
yoksis_birim = client.bucket_type('catalog').bucket('ulakbus_yoksis_birim')
y = yoksis_birim.get(str(birim_id))
data = self.birim_detaylari()
y.data = data
# print("%s: stored.." % (birim_id))
self.logger.info("%s icin kaydedildi: \n\n" % birim_id)
y.store()
def birim_kaydet(self, birim_id):
# self.kvdb.conn.set(birim_id, self.birim_detaylari())
# self.logger.info("%s icin degerler: %s\n\n" % (birim_id, self.birim_detaylari()))
from pyoko.db.connection import client
yoksis_birim = client.bucket_type('catalog').bucket(
'ulakbus_yoksis_birim')
y = yoksis_birim.get(str(birim_id))
data = self.birim_detaylari()
y.data = data
# print("%s: stored.." % (birim_id))
self.logger.info("%s icin kaydedildi: \n\n" % birim_id)
y.store()
def run(self):
from pyoko.db.connection import client
import os
fixture_bucket = client.bucket_type('catalog').bucket('ulakbus_settings_fixtures')
path = self.manager.args.path
if os.path.isdir(path):
from glob import glob
for fixture_file in glob(os.path.join(path, "*.json")):
self.dump(fixture_file, fixture_bucket)
else:
self.dump(path, fixture_bucket)
def run(self):
from pyoko.db.connection import client
import os
fixture_bucket = client.bucket_type('catalog').bucket('ulakbus_settings_fixtures')
path = self.manager.args.path
if os.path.isdir(path):
from glob import glob
for fixture_file in glob(os.path.join(path, "*.json")):
self.dump(fixture_file, fixture_bucket)
else:
self.dump(path, fixture_bucket)
def test_count(self):
mb = client.bucket_type('pyoko_models').bucket('student')
Student.objects._clear()
results = mb.search('-deleted:True', 'pyoko_models_student',
**{'rows': 0})
assert Student.objects.count() == results['num_found'] == 0
# 770 records will be saved.
for i in range(770):
Student(number=str(i % 3)).save()
# wait until 770 records are saved.
while mb.search('-deleted:True', 'pyoko_models_student', **
{'rows': 0})['num_found'] != 770:
time.sleep(0.3)
# total count
assert Student.objects.count() == 770
# number '2' results count
results = mb.search('-deleted:True AND number:2',
'pyoko_models_student', **{'rows': 0})
assert Student.objects.filter(
number='2').count() == results['num_found'] == 256
# total count
assert Student.objects.filter(
number='2').count() + Student.objects.filter(
number='1').count() + Student.objects.filter(
number='0').count() == 770
# set_params and count tests:
assert Student.objects.filter(number='2').set_params(
start=0).count() == 256
assert Student.objects.filter(number='2').set_params(
start=0, rows=35).count() == 35
assert Student.objects.filter(number='2').set_params(
start=125, rows=35).count() == 35
assert Student.objects.filter(number='2').set_params(
start=0, rows=0).count() == 0
assert Student.objects.filter(number='2').set_params(
rows=100).count() == 100
assert Student.objects.filter(number='2').set_params(
start=250, rows=100).count() == 6
assert Student.objects.filter(number='2').set_params(
start=300, rows=100).count() == 0
self.prepare_testbed(reset=True)
def find_models_and_delete_search_index(self, model, force, exec_models,
check_only):
"""
Finds models to execute and these models' exist search indexes are deleted.
For other operations, necessary models are gathered to list(exec_models)
Args:
model: model to execute
force(bool): True or False if True, all given models are executed.
exec_models(list): if not force, models to execute are gathered to list.
If there is not necessity to migrate operation model doesn't put to exec list.
check_only: do not migrate, only report migration is needed or not if True
Returns:
"""
ins = model(fake_context)
fields = self.get_schema_fields(ins._collect_index_fields())
new_schema = self.compile_schema(fields)
bucket_name = model._get_bucket_name()
bucket_type = client.bucket_type(settings.DEFAULT_BUCKET_TYPE)
bucket = bucket_type.bucket(bucket_name)
index_name = "%s_%s" % (settings.DEFAULT_BUCKET_TYPE, bucket_name)
if not force:
try:
schema = get_schema_from_solr(index_name)
if schema == new_schema:
print("Schema %s is already up to date, nothing to do!" %
index_name)
return
elif check_only and schema != new_schema:
print("Schema %s is not up to date, migrate this model!" %
index_name)
return
except:
import traceback
traceback.print_exc()
bucket.set_property('search_index', 'foo_index')
try:
client.delete_search_index(index_name)
except RiakError as e:
if 'notfound' != e.value:
raise
wait_for_schema_deletion(index_name)
exec_models.append(model)
def test_all(self):
mb = client.bucket_type('pyoko_models').bucket('student')
row_size = BaseAdapter()._cfg['row_size']
Student.objects._clear()
assert Student.objects.count() == 0
for i in range(row_size + 100):
Student(name=str(i)).save()
while Student.objects.count() != row_size + 100:
time.sleep(0.3)
# Wanted result from filter method much than default row_size.
# It should raise an exception.
with pytest.raises(Exception):
Student.objects.filter()
# Results are taken from solr in ordered with 'timestamp' sort parameter.
results = mb.search(
'-deleted:True', 'pyoko_models_student', **{
'sort': 'timestamp desc',
'fl': '_yz_rk, score',
'rows': row_size + 100
})
# Ordered key list is created.
ordered_key_list = [doc['_yz_rk'] for doc in results['docs']]
# Getting data from riak with unordered way is tested.
students = Student.objects.all()
assert len(students) == row_size + 100
assert students.adapter.ordered == False
# Getting data from riak with ordered way is tested.
temp_key_list = []
students = Student.objects.order_by().all()
assert students.adapter.ordered == True
for student in students:
temp_key_list.append(student.key)
assert len(temp_key_list) == row_size + 100
assert temp_key_list == ordered_key_list
self.prepare_testbed(reset=True)
inline_edit = ['catalog_key', 'tr', 'en']
# we do NOT want checkboxes on right of the ui table view
allow_selection = False
# set meta translate_widget to True to use translate view for ui
translate_widget = True
save = fields.Button("Save", cmd="save_catalog", flow="start")
cancel = fields.Button("Cancel", cmd="cancel", flow="start")
class CatalogDatas(ListNode):
catalog_key = fields.String()
tr = fields.String("Türkçe")
en = fields.String("English")
fixture_bucket = client.bucket_type('catalog').bucket(
'ulakbus_settings_fixtures')
class CatalogDataView(CrudView):
"""
Workflow class of catalog add/edit screens
"""
def list_catalogs(self):
"""
Lists existing catalogs respect to ui view template format
"""
_form = CatalogSelectForm(current=self.current)
_form.set_choices_of('catalog',
[(i, i) for i in fixture_bucket.get_keys()])
self.form_out(_form)
def apply_schema(client, force, job_pack, check_only):
"""
riak doesn't support schema/index updates ( http://git.io/vLOTS )
as a workaround, we create a temporary index,
attach it to the bucket, delete the old index/schema,
re-create the index with new schema, assign it to bucket,
then delete the temporary index.
:param byte new_schema: compiled schema
:param str bucket_name: name of schema, index and bucket.
:return: True or False
:rtype: bool
"""
for new_schema, model in job_pack:
try:
bucket_name = model._get_bucket_name()
bucket_type = client.bucket_type(settings.DEFAULT_BUCKET_TYPE)
bucket = bucket_type.bucket(bucket_name)
n_val = bucket_type.get_property('n_val')
index_name = "%s_%s" % (settings.DEFAULT_BUCKET_TYPE, bucket_name)
if not force:
try:
schema = get_schema_from_solr(index_name)
if schema == new_schema:
print("Schema %s is already up to date, nothing to do!" % index_name)
continue
elif check_only and schema != new_schema:
print("Schema %s is not up to date, migrate this model!" % index_name)
continue
except:
import traceback
traceback.print_exc()
bucket.set_property('search_index', 'foo_index')
try:
client.delete_search_index(index_name)
except RiakError as e:
if 'notfound' != e.value:
raise
wait_for_schema_deletion(index_name)
client.create_search_schema(index_name, new_schema)
client.create_search_index(index_name, index_name, n_val)
bucket.set_property('search_index', index_name)
print("+ %s (%s)" % (model.__name__, index_name))
stream = bucket.stream_keys()
i = 0
unsaved_keys = []
for key_list in stream:
for key in key_list:
i += 1
# time.sleep(0.4)
try:
obj = bucket.get(key)
if obj.data:
obj.store()
except ConflictError:
unsaved_keys.append(key)
print("Error on save. Record in conflict: %s > %s" % (bucket_name, key))
except:
unsaved_keys.append(key)
print("Error on save! %s > %s" % (bucket_name, key))
import traceback
traceback.print_exc()
stream.close()
print("Re-indexed %s records of %s" % (i, bucket_name))
if unsaved_keys:
print("\nThese keys cannot be updated:\n\n", unsaved_keys)
except:
print("n_val: %s" % n_val)
print("bucket_name: %s" % bucket_name)
print("bucket_type: %s" % bucket_type)
raise
def apply_schema(client, force, job_pack):
"""
riak doesn't support schema/index updates ( http://git.io/vLOTS )
as a workaround, we create a temporary index,
attach it to the bucket, delete the old index/schema,
re-create the index with new schema, assign it to bucket,
then delete the temporary index.
:param byte new_schema: compiled schema
:param str bucket_name: name of schema, index and bucket.
:return: True or False
:rtype: bool
"""
for new_schema, model in job_pack:
try:
bucket_name = model._get_bucket_name()
bucket_type = client.bucket_type(settings.DEFAULT_BUCKET_TYPE)
bucket = bucket_type.bucket(bucket_name)
n_val = bucket_type.get_property('n_val')
index_name = "%s_%s" % (settings.DEFAULT_BUCKET_TYPE,
bucket_name)
if not force:
try:
if get_schema_from_solr(index_name) == new_schema:
print(
"Schema %s already up to date, nothing to do!"
% index_name)
continue
except:
import traceback
traceback.print_exc()
bucket.set_property('search_index', 'foo_index')
try:
client.delete_search_index(index_name)
except RiakError as e:
if 'notfound' != e.value:
raise
wait_for_schema_deletion(index_name)
client.create_search_schema(index_name, new_schema)
client.create_search_index(index_name, index_name, n_val)
bucket.set_property('search_index', index_name)
print("+ %s (%s)" % (model.__name__, index_name))
stream = bucket.stream_keys()
i = 0
unsaved_keys = []
for key_list in stream:
for key in key_list:
i += 1
# time.sleep(0.4)
try:
obj = bucket.get(key)
if obj.data:
obj.store()
except ConflictError:
unsaved_keys.append(key)
print(
"Error on save. Record in conflict: %s > %s" %
(bucket_name, key))
except:
unsaved_keys.append(key)
print("Error on save! %s > %s" %
(bucket_name, key))
import traceback
traceback.print_exc()
stream.close()
print("Re-indexed %s records of %s" % (i, bucket_name))
if unsaved_keys:
print("\nThese keys cannot be updated:\n\n", unsaved_keys)
except:
print("n_val: %s" % n_val)
print("bucket_name: %s" % bucket_name)
print("bucket_type: %s" % bucket_type)
raise
def get_from_db(self, cat):
from pyoko.db.connection import client
data = client.bucket_type('catalog').bucket(
'ulakbus_settings_fixtures').get(cat).data
return self.parse_db_data(data, cat)
def get_from_db(self, cat):
from pyoko.db.connection import client
data = client.bucket_type('catalog').bucket('ulakbus_settings_fixtures').get(cat).data
return self.parse_db_data(data, cat)
