def test_no_cqrs_queue(settings):
settings.CQRS.update({'queue': None})
with pytest.raises(AssertionError) as e:
ReplicaRegistry.register_model(replica_models.MappedFieldsModelRef)
assert str(
e.value
) == 'CQRS queue must be set for the service, that has replica models.'
def __new__(mcs, *args):
model_cls = super(ReplicaMeta, mcs).__new__(mcs, *args)
if args[0] != 'ReplicaMixin':
_MetaUtils.check_cqrs_id(model_cls)
ReplicaMeta._check_cqrs_mapping(model_cls)
ReplicaRegistry.register_model(model_cls)
return model_cls
def handle(self, *args, **options):
batch_size = self._get_batch_size(options)
f_name = options['input']
if f_name != '-' and not os.path.exists(f_name):
raise CommandError("File {0} doesn't exist!".format(f_name))
with sys.stdin if f_name == '-' else open(f_name, 'r') as f:
try:
cqrs_id = next(f).strip()
except StopIteration:
cqrs_id = None
if not cqrs_id:
raise CommandError('File {0} is empty!'.format(f_name))
model = ReplicaRegistry.get_model_by_cqrs_id(cqrs_id)
if not model:
raise CommandError('Wrong CQRS ID: {0}!'.format(cqrs_id))
with transaction.atomic():
if options['clear']:
try:
model._default_manager.all().delete()
except DatabaseError:
raise CommandError("Delete operation fails!")
self._process(f, model, batch_size)
def _get_model(options):
cqrs_id = options['cqrs_id']
model = ReplicaRegistry.get_model_by_cqrs_id(cqrs_id)
if not model:
raise CommandError('Wrong CQRS ID: {}!'.format(cqrs_id))
return model
def _get_model(first_line):
cqrs_id = first_line.split(',')[0]
model = ReplicaRegistry.get_model_by_cqrs_id(cqrs_id)
if not model:
raise CommandError('Wrong CQRS ID: {}!'.format(cqrs_id))
return model
def _fail_message(cls, channel, delivery_tag, payload, exception,
delay_queue):
cls.log_consumed_failed(payload)
model_cls = ReplicaRegistry.get_model_by_cqrs_id(payload.cqrs_id)
if model_cls is None:
logger.error("Model for cqrs_id {0} is not found.".format(
payload.cqrs_id))
cls._nack(channel, delivery_tag)
return
if model_cls.should_retry_cqrs(payload.retries, exception):
delay = model_cls.get_cqrs_retry_delay(payload.retries)
cls._delay_message(channel, delivery_tag, payload, delay,
delay_queue)
else:
cls._add_to_dead_letter_queue(channel, payload)
cls._nack(channel, delivery_tag)
def route_signal_to_replica_model(signal_type,
cqrs_id,
instance_data,
previous_data=None):
""" Routes signal to model method to create/update/delete replica instance.
:param dj_cqrs.constants.SignalType signal_type: Consumed signal type.
:param str cqrs_id: Replica model CQRS unique identifier.
:param dict instance_data: Master model data.
"""
if signal_type not in (SignalType.DELETE, SignalType.SAVE,
SignalType.SYNC):
logger.error('Bad signal type "{}" for CQRS_ID "{}".'.format(
signal_type, cqrs_id))
return
model_cls = ReplicaRegistry.get_model_by_cqrs_id(cqrs_id)
if model_cls:
db_is_needed = not model_cls.CQRS_NO_DB_OPERATIONS
if db_is_needed:
close_old_connections()
with transaction.atomic(
savepoint=False) if db_is_needed else ExitStack():
if signal_type == SignalType.DELETE:
return model_cls.cqrs_delete(instance_data)
elif signal_type == SignalType.SAVE:
return model_cls.cqrs_save(instance_data,
previous_data=previous_data)
elif signal_type == SignalType.SYNC:
return model_cls.cqrs_save(
instance_data,
previous_data=previous_data,
sync=True,
)
def test_get_model_by_cqrs_id_no_id(caplog):
assert ReplicaRegistry.get_model_by_cqrs_id('invalid') is None
assert 'No model with such CQRS_ID: invalid.' in caplog.text
def handle(self, *args, **options):
batch_size = self._get_batch_size(options)
f_name = options['input']
if f_name != '-' and not os.path.exists(f_name):
raise CommandError("File {} doesn't exist!".format(f_name))
with sys.stdin if f_name == '-' else open(f_name, 'r') as f:
try:
cqrs_id = next(f).strip()
except StopIteration:
cqrs_id = None
if not cqrs_id:
raise CommandError('File {} is empty!'.format(f_name))
model = ReplicaRegistry.get_model_by_cqrs_id(cqrs_id)
if not model:
raise CommandError('Wrong CQRS ID: {}!'.format(cqrs_id))
success_counter = 0
with transaction.atomic():
if options['clear']:
try:
model._default_manager.all().delete()
except DatabaseError:
raise CommandError("Delete operation fails!")
line_number = 2
while True:
with transaction.atomic():
try:
for _ in range(0, batch_size):
line = f.readline()
if not line:
raise EOFError
try:
try:
master_data = ujson.loads(line.strip())
except ValueError:
print(
"Dump file can't be parsed: line {}!".
format(line_number, ),
file=sys.stderr)
line_number += 1
continue
instance = model.cqrs_save(master_data)
if not instance:
print("Instance can't be saved: line {}!".
format(line_number),
file=sys.stderr)
else:
success_counter += 1
except Exception as e:
print('Unexpected error: line {}! {}'.format(
line_number, str(e)),
file=sys.stderr)
line_number += 1
except EOFError:
break
print('Done!\n{} instance(s) loaded.'.format(success_counter),
file=sys.stderr)