def Process(self, request, context=None):
event_id = request.event_id
event_service_instance_id = request.event_service_instance_id
logger.debug(
'processor_id: %s received process request on event_id: %s with event_service_instance_id: %s',
self._runner.processor_id, event_id, event_service_instance_id)
params = {}
_structs.copy_struct_to_dict(request.params, params)
try:
response = processing_pb2.ProcessResponse()
result, times, added_indices = self._runner.call_process(
event_id, event_service_instance_id, params)
if result is not None:
_structs.copy_dict_to_struct(result, response.result, [])
_timing.add_times(self._times_map, times)
for k, l in times.items():
response.timing_info[k].FromTimedelta(l)
for document_name, l in added_indices.items():
for index_name in l:
created_index = response.created_indices.add()
created_index.document_name = document_name
created_index.index_name = index_name
return response
except Exception as e:
logger.error(str(e))
logger.error(traceback.format_exc())
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(str(e))
return empty_pb2.Empty()
def test_none_list_field_reverse():
struct = struct_pb2.Struct()
bar = struct.get_or_create_list('bar')
bar.append(None)
d = {}
copy_struct_to_dict(struct, d)
assert d['bar'] == [None]
def test_list_list_field_reverse():
struct = struct_pb2.Struct()
bar = struct.get_or_create_list('bar')
bar.add_list().extend([1, 2, 3])
bar.add_list().extend([4, 5, 6])
d = {}
copy_struct_to_dict(struct, d)
assert d['bar'] == [[1, 2, 3], [4, 5, 6]]
def test_struct_list_field_reverse():
struct = struct_pb2.Struct()
bar = struct.get_or_create_list('bar')
bar.add_struct()['bar'] = 'baz'
bar.add_struct()['foo'] = 'bar'
d = {}
copy_struct_to_dict(struct, d)
assert d['bar'] == [{'bar': 'baz'}, {'foo': 'bar'}]
def create_index_from_response(self, response):
json_labels = response.json_labels
labels = []
for label in json_labels.labels:
d = {}
_structs.copy_struct_to_dict(label, d)
generic_label = GenericLabel(**d)
labels.append(generic_label)
return label_index(labels, json_labels.is_distinct)
def create_index_from_response(self, response):
generic_labels = response.generic_labels
labels = []
for label_message in generic_labels.labels:
fields = {}
_structs.copy_struct_to_dict(label_message.fields, fields)
reference_field_ids = {}
_structs.copy_struct_to_dict(label_message.reference_ids,
reference_field_ids)
generic_label = _labels.GenericLabel(
label_message.start_index,
label_message.end_index,
identifier=label_message.identifier,
fields=fields,
reference_field_ids=reference_field_ids)
labels.append(generic_label)
return _label_indices.presorted_label_index(labels,
generic_labels.is_distinct,
adapter=self)
def call_process(self, event_id, event_instance_id, params):
logger.debug(
'processor_id: %s calling process on event_id: %s with event_instance_id: %s',
self.processor_id, event_id, event_instance_id)
p = dict(self.params or {})
if params is not None:
p.update(params)
with _base.Processor.enter_context() as context:
request = processing_pb2.ProcessRequest(
processor_id=self.processor_id,
event_id=event_id,
event_service_instance_id=event_instance_id)
_structs.copy_dict_to_struct(p, request.params, [p])
with _base.Processor.started_stopwatch('remote_call'):
try:
response = self._stub.Process(request)
except Exception as e:
logger.error(
"Error while processing event_id %s through remote pipeline "
"component '%s'", event_id, self.component_id)
raise e
r = {}
_structs.copy_struct_to_dict(response.result, r)
timing_info = response.timing_info
for k, v in timing_info.items():
context.add_time(k, v.ToTimedelta())
created_indices = {}
for created_index in response.created_indices:
try:
doc_created_indices = created_indices[
created_index.document_name]
except KeyError:
doc_created_indices = []
created_indices[
created_index.document_name] = doc_created_indices
doc_created_indices.append(created_index.index_name)
return r, context.times, created_indices
def call_process(self, event_id, params):
self.processed += 1
p = dict(self.params or {})
if params is not None:
p.update(params)
with EventProcessor.enter_context() as context:
try:
request = processing_pb2.ProcessRequest(
processor_id=self._processor_id, event_id=event_id)
_structs.copy_dict_to_struct(p, request.params, [p])
with Processor.started_stopwatch('remote_call'):
response = self._stub.Process(request)
r = {}
_structs.copy_struct_to_dict(response.result, r)
timing_info = response.timing_info
for k, v in timing_info.items():
context.add_time(k, v.ToTimedelta())
created_indices = {}
for created_index in response.created_indices:
try:
doc_created_indices = created_indices[
created_index.document_name]
except KeyError:
doc_created_indices = []
created_indices[
created_index.document_name] = doc_created_indices
doc_created_indices.append(created_index.index_name)
return r, context.times, created_indices
except Exception as e:
self.failure_count += 1
logger.error(
'Processor "%s" failed while processing event with id: %s',
self.component_id, event_id)
logger.error(e)
raise e
def test_empty_struct():
struct = struct_pb2.Struct()
d = {}
copy_struct_to_dict(struct, d)
assert d == {}
def test_str_list_field_reverse():
struct = struct_pb2.Struct()
struct.get_or_create_list('bar').extend(['a', 'b', 'c'])
d = {}
copy_struct_to_dict(struct, d)
assert d['bar'] == ['a', 'b', 'c']
def test_struct_field_reverse():
struct = struct_pb2.Struct()
struct.get_or_create_struct('bar')['bar'] = 'baz'
d = {}
copy_struct_to_dict(struct, d)
assert d['bar'] == {'bar': 'baz'}
def test_null_field_reverse():
struct = struct_pb2.Struct()
struct['bar'] = None
d = {}
copy_struct_to_dict(struct, d)
assert d == {'bar': None}
def test_bool_field_reverse():
struct = struct_pb2.Struct()
struct['bar'] = True
d = {}
copy_struct_to_dict(struct, d)
assert d['bar'] is True
def test_int_field_reverse():
struct = struct_pb2.Struct()
struct['bar'] = 1
d = {}
copy_struct_to_dict(struct, d)
assert d['bar'] == 1
def test_int_list_field_reverse():
struct = struct_pb2.Struct()
struct.get_or_create_list('bar').extend([1, 2, 3])
d = {}
copy_struct_to_dict(struct, d)
assert d['bar'] == [1, 2, 3]
def test_bool_list_field_reverse():
struct = struct_pb2.Struct()
struct.get_or_create_list('bar').extend([True, False, True])
d = {}
copy_struct_to_dict(struct, d)
assert d['bar'] == [True, False, True]
def test_empty_list_field_reverse():
struct = struct_pb2.Struct()
struct.get_or_create_list('bar')
d = {}
copy_struct_to_dict(struct, d)
assert d['bar'] == []