def test_operation_bulk_index(client, es_clear, operation_log_data):
"""Test operation logs bulk creation."""
data = []
for date in [
'2020-01-21T09:51:52.879533+00:00',
'2020-02-21T09:51:52.879533+00:00',
'2020-03-21T09:51:52.879533+00:00',
'2020-04-21T09:51:52.879533+00:00',
'2021-01-21T09:51:52.879533+00:00',
'2021-02-21T09:51:52.879533+00:00'
]:
tmp = deepcopy(operation_log_data)
tmp['date'] = date
data.append(tmp)
OperationLog.bulk_index(data)
# flush the index for the test
current_search.flush_and_refresh(OperationLog.index_name)
assert OperationLog.get_indices() == set((
'operation_logs-2020',
'operation_logs-2021'
))
with pytest.raises(Exception) as excinfo:
data[0]['operation'] = dict(name='foo')
OperationLog.bulk_index(data)
assert "BulkIndexError" in str(excinfo.value)
# clean up the index
assert OperationLog.delete_indices()
def create_operation_logs(infile, lazy, size):
"""Load operation log records in reroils.
:param infile: Json operation log file.
:param lazy: lazy reads file
"""
click.secho('Load operation log records:', fg='green')
if lazy:
# try to lazy read json file (slower, better memory management)
data = read_json_record(infile)
else:
# load everything in memory (faster, bad memory management)
data = json.load(infile)
index_count = 0
with click.progressbar(data) as bar:
records = []
for oplg in bar:
if not (index_count + 1) % size:
OperationLog.bulk_index(records)
records = []
records.append(oplg)
index_count += 1
# the rest of the records
if records:
OperationLog.bulk_index(records)
index_count += len(records)
click.echo(f'created {index_count} operation logs.')