def handle(self, *args, **options):
model = self._get_model(options)
batch_size = self._get_batch_size(options)
qs = model._default_manager.all().order_by()
if options['filter']:
try:
kwargs = ujson.loads(options['filter'])
if not isinstance(kwargs, dict):
raise ValueError
except ValueError:
raise CommandError('Bad filter kwargs!')
try:
qs = qs.filter(**kwargs)
except FieldError as e:
raise CommandError('Bad filter kwargs! {0}'.format(str(e)))
if not qs.exists():
self.stderr.write('No objects found for filter!')
return
current_dt = now()
self.stdout.write('{0},{1}'.format(model.CQRS_ID, str(current_dt)))
for bqs in batch_qs(qs, batch_size=batch_size):
package = [[instance.pk, instance.cqrs_revision]
for instance in bqs if instance.is_sync_instance()]
self.stdout.write(self.serialize_package(package))
def handle(self, *args, **options):
model = self._get_model(options)
batch_size = self._get_batch_size(options)
qs = model._default_manager.values().order_by()
if options['filter']:
try:
kwargs = ujson.loads(options['filter'])
if not isinstance(kwargs, dict):
raise ValueError
except ValueError:
raise CommandError('Bad filter kwargs!')
try:
qs = qs.filter(**kwargs)
except FieldError as e:
raise CommandError('Bad filter kwargs! {}'.format(str(e)))
if not qs.exists():
self.stderr.write('No objects found for filter!')
return
current_dt = now()
self.stdout.write('{},{}'.format(model.CQRS_ID, str(current_dt)))
for bqs in batch_qs(qs.values_list('pk', flat=True),
batch_size=batch_size):
self.stdout.write(self.serialize_package(list(bqs)))
def handle(self, *args, **options):
model = self._get_model(options)
progress = self._get_progress(options)
batch_size = self._get_batch_size(options)
qs = self._prepare_qs(model, options)
db_count = qs.count()
if db_count == 0:
print('No objects found for filter!')
return
counter, success_counter = 0, 0
if progress:
print('Processing {} records with batch size {}'.format(
db_count, batch_size))
for qs in batch_qs(model.relate_cqrs_serialization(qs),
batch_size=batch_size):
ts = time.time()
cs = counter
for instance in qs:
counter += 1
try:
instance.cqrs_sync(queue=options['queue'])
success_counter += 1
except Exception as e:
print('\nSync record failed for pk={}: {}: {}'.format(
instance.pk,
type(e).__name__,
str(e),
))
if progress:
rate = (counter - cs) / (time.time() - ts)
percent = 100 * counter / db_count
eta = datetime.timedelta(seconds=int((db_count - counter) /
rate))
sys.stdout.write(
'\r{} of {} processed - {}% with rate {:.1f} rps, to go {} ...{:20}'
.format(
counter,
db_count,
int(percent),
rate,
str(eta),
' ',
))
sys.stdout.flush()
print(
'Done!\n{} instance(s) synced.\n{} instance(s) processed.'.format(
success_counter,
counter,
))
def handle(self, *args, **options):
model = self._get_model(options)
out_fname = self._get_output_filename(options)
progress = self._get_progress(options)
batch_size = self._get_batch_size(options)
with sys.stdout if out_fname == '-' else open(out_fname, 'w') as f:
f.write(model.CQRS_ID)
counter, success_counter = 0, 0
db_count = model._default_manager.count()
if progress:
print(
'Processing {} records with batch size {}'.format(db_count, batch_size),
file=sys.stderr,
)
for qs in batch_qs(
model.relate_cqrs_serialization(model._default_manager.order_by().all()),
batch_size=batch_size,
):
ts = time.time()
cs = counter
for instance in qs:
counter += 1
try:
f.write(
'\n' + ujson.dumps(instance.to_cqrs_dict()),
)
success_counter += 1
except Exception as e:
print('\nDump record failed for pk={}: {}: {}'.format(
instance.pk, type(e).__name__, str(e),
), file=sys.stderr)
if progress:
rate = (counter - cs) / (time.time() - ts)
percent = 100 * counter / db_count
eta = datetime.timedelta(seconds=int((db_count - counter) / rate))
sys.stderr.write(
'\r{} of {} processed - {}% with rate {:.1f} rps, to go {} ...{:20}'.format(
counter, db_count, int(percent), rate, str(eta), ' ',
))
sys.stderr.flush()
print('Done!\n{} instance(s) saved.\n{} instance(s) processed.'.format(
success_counter, counter,
), file=sys.stderr)
def handle(self, *args, **options):
model = self._get_model(options)
progress = self._get_progress(options)
batch_size = self._get_batch_size(options)
qs = model._default_manager.none()
if options['filter']:
try:
kwargs = ujson.loads(options['filter'])
if not isinstance(kwargs, dict):
raise ValueError
except ValueError:
raise CommandError('Bad filter kwargs!')
try:
qs = model._default_manager.filter(**kwargs).order_by()
except FieldError as e:
raise CommandError('Bad filter kwargs! {}'.format(str(e)))
db_count = qs.count()
if db_count == 0:
print('No objects found for filter!')
return
counter, success_counter = 0, 0
if progress:
print('Processing {} records with batch size {}'.format(
db_count, batch_size))
for qs in batch_qs(model.relate_cqrs_serialization(qs),
batch_size=batch_size):
ts = time.time()
cs = counter
for instance in qs:
counter += 1
try:
instance.cqrs_sync(queue=options['queue'])
success_counter += 1
except Exception as e:
print('\nSync record failed for pk={}: {}: {}'.format(
instance.pk,
type(e).__name__,
str(e),
))
if progress:
rate = (counter - cs) / (time.time() - ts)
percent = 100 * counter / db_count
eta = datetime.timedelta(seconds=int((db_count - counter) /
rate))
sys.stdout.write(
'\r{} of {} processed - {}% with rate {:.1f} rps, to go {} ...{:20}'
.format(
counter,
db_count,
int(percent),
rate,
str(eta),
' ',
))
sys.stdout.flush()
print(
'Done!\n{} instance(s) synced.\n{} instance(s) processed.'.format(
success_counter,
counter,
))