def save_to_elastic(signal_id):
if not SignalDocument.ping():
raise Exception('Elastic cluster is unreachable')
signal = Signal.objects.get(id=signal_id)
signal_document = SignalDocument.create_document(signal)
signal_document.save()
def _index_date_range(self, from_date=None, to_date=None):
if not from_date and not to_date:
self.stderr.write(
'* At least a from date or a to date needs to be specified')
return
if from_date:
from_date = timezone.make_aware(
timezone.datetime.strptime(from_date, '%Y-%m-%d'))
else:
from_date = timezone.now()
from_date = from_date.replace(hour=00, minute=00,
second=00) # Beginning of the given day
if to_date:
to_date = timezone.make_aware(
timezone.datetime.strptime(to_date, '%Y-%m-%d'))
else:
to_date = timezone.now()
to_date = to_date + timezone.timedelta(days=1)
to_date = to_date.replace(hour=00, minute=00,
second=00) # Beginning of the next day
if to_date < from_date:
self.stderr.write('* To date cannot be before the from date')
return
signal_qs = Signal.objects.filter(
updated_at__range=[from_date, to_date]).order_by('-updated_at')
self.stdout.write(
f'* Indexing Signals in range from {from_date:%Y-%m-%d %H:%M:%S} to '
f'{to_date:%Y-%m-%d %H:%M:%S}, found {signal_qs.count()} signals')
if not self._dry_run:
SignalDocument.index_documents(queryset=signal_qs)
def rebuild_index():
if settings.FEATURE_FLAGS.get('SEARCH_BUILD_INDEX', False):
log.info('rebuild_index - start')
SignalDocument.index_documents()
log.info('rebuild_index - done!')
else:
log.warning('rebuild_index - elastic indexing disabled')
def rebuild_index():
log.info('rebuild_index - start')
if not SignalDocument.ping():
raise Exception('Elastic cluster is unreachable')
SignalDocument.index_documents()
log.info('rebuild_index - done!')
def handle(self, *args, **options):
if settings.FEATURE_FLAGS.get('SEARCH_BUILD_INDEX', False):
self.stdout.write('elastic indexing')
delete = options['delete']
if delete:
SignalDocument.clear_index()
SignalDocument.index_documents()
else:
self.stdout.write('elastic indexing disabled')
self.stdout.write('done!')
def delete_from_elastic(signal):
if not SignalDocument.ping():
raise Exception('Elastic cluster is unreachable')
if isinstance(signal, int):
signal = Signal.objects.get(id=signal)
signal_document = SignalDocument.create_document(signal)
try:
signal_document.delete()
except NotFoundError:
log.warning(f'Signal {signal.id} not found in Elasticsearch')
def _index_signal(self, signal_id):
self.stdout.write(f'* Index Signal #{signal_id}')
try:
signal = Signal.objects.get(pk=signal_id)
except Signal.DoesNotExist:
self.stderr.write(f'* Signal #{signal_id} does not exists')
return None
if not self._dry_run:
SignalDocument.init()
signal_document = SignalDocument.create_document(signal)
signal_document.save()
def _index_signals(self, signal_ids):
self.stdout.write(
f'* Indexing Signals: #{", #".join(map(str, signal_ids))}')
signal_qs = Signal.objects.filter(
id__in=signal_ids).order_by('-updated_at')
signal_qs_ids = signal_qs.values_list('id', flat=True)
ids_diff = set(signal_ids) - set(signal_qs_ids)
if ids_diff:
self.stderr.write(
f'* Signals not found #{", #".join(map(str, ids_diff))}')
else:
if not self._dry_run:
SignalDocument.index_documents(queryset=signal_qs)
def save_to_elastic(signal_id):
if settings.FEATURE_FLAGS.get('SEARCH_BUILD_INDEX', False):
signal = Signal.objects.get(id=signal_id)
signal_document = SignalDocument.create_document(signal)
signal_document.save()
else:
log.warning('rebuild_index - elastic indexing disabled')
def handle(self, *args, **options):
start = timer()
self.stdout.write('Elastic management command')
ping = True if self._dry_run else SignalDocument.ping()
if not ping:
self.stderr.write('* Elastic cluster is unreachable!')
else:
self._apply_options(**options)
stop = timer()
self.stdout.write(f'Time: {stop - start:.2f} second(s)')
self.stdout.write('Done!')
def get_queryset(self, *args, **kwargs):
if settings.FEATURE_FLAGS.get('API_SEARCH_ENABLED', False):
if 'q' in self.request.query_params:
q = self.request.query_params['q']
else:
q = '*'
multi_match = MultiMatch(
query=q,
fields=['id', 'text', 'category_assignment.category.name'])
s = SignalDocument.search().query(multi_match)
s.execute()
return s
else:
raise NotImplementedException('Not implemented')
def get_queryset(self, *args, **kwargs):
if 'q' in self.request.query_params:
q = self.request.query_params['q']
else:
q = '*'
multi_match = MultiMatch(
query=q,
fields=[
'id',
'text',
'category_assignment.category.name',
'reporter.email', # SIG-2058 [BE] email, telefoon aan vrij zoeken toevoegen
'reporter.phone' # SIG-2058 [BE] email, telefoon aan vrij zoeken toevoegen
])
s = SignalDocument.search().query(multi_match)
s.execute()
return s
def get_queryset(self, *args, **kwargs):
if settings.FEATURE_FLAGS.get('API_SEARCH_ENABLED', False):
if 'q' in self.request.query_params:
q = self.request.query_params['q']
else:
q = '*'
multi_match = MultiMatch(
query=q,
fields=[
'id',
'text',
'category_assignment.category.name',
'reporter.email', # SIG-2058 [BE] email, telefoon aan vrij zoeken toevoegen
'reporter.phone' # SIG-2058 [BE] email, telefoon aan vrij zoeken toevoegen
])
s = SignalDocument.search().query(multi_match)
s.execute()
return s
else:
raise NotImplementedException('Not implemented')
def list(self, request, *args, **kwargs):
if not SignalDocument.ping():
raise GatewayTimeoutException(
detail='The elastic cluster is unreachable')
return super().list(request=request, *args, **kwargs)
def _clear_index(self):
self.stdout.write('* Clear the index')
if not self._dry_run:
SignalDocument.clear_index()
self._init_index()
def _init_index(self):
self.stdout.write('* Init the index')
if not self._dry_run:
SignalDocument.init()
def _index_documents(self):
self.stdout.write('* Index all Signals in bulk')
if not self._dry_run:
SignalDocument.index_documents()