def Process(self, request, context=None): event_id = request.event_id event_service_instance_id = request.event_service_instance_id logger.debug( 'processor_id: %s received process request on event_id: %s with event_service_instance_id: %s', self._runner.processor_id, event_id, event_service_instance_id) params = {} _structs.copy_struct_to_dict(request.params, params) try: response = processing_pb2.ProcessResponse() result, times, added_indices = self._runner.call_process( event_id, event_service_instance_id, params) if result is not None: _structs.copy_dict_to_struct(result, response.result, []) _timing.add_times(self._times_map, times) for k, l in times.items(): response.timing_info[k].FromTimedelta(l) for document_name, l in added_indices.items(): for index_name in l: created_index = response.created_indices.add() created_index.document_name = document_name created_index.index_name = index_name return response except Exception as e: logger.error(str(e)) logger.error(traceback.format_exc()) context.set_code(grpc.StatusCode.INTERNAL) context.set_details(str(e)) return empty_pb2.Empty()
def test_none_list_field_reverse(): struct = struct_pb2.Struct() bar = struct.get_or_create_list('bar') bar.append(None) d = {} copy_struct_to_dict(struct, d) assert d['bar'] == [None]
def test_list_list_field_reverse(): struct = struct_pb2.Struct() bar = struct.get_or_create_list('bar') bar.add_list().extend([1, 2, 3]) bar.add_list().extend([4, 5, 6]) d = {} copy_struct_to_dict(struct, d) assert d['bar'] == [[1, 2, 3], [4, 5, 6]]
def test_struct_list_field_reverse(): struct = struct_pb2.Struct() bar = struct.get_or_create_list('bar') bar.add_struct()['bar'] = 'baz' bar.add_struct()['foo'] = 'bar' d = {} copy_struct_to_dict(struct, d) assert d['bar'] == [{'bar': 'baz'}, {'foo': 'bar'}]
def create_index_from_response(self, response): json_labels = response.json_labels labels = [] for label in json_labels.labels: d = {} _structs.copy_struct_to_dict(label, d) generic_label = GenericLabel(**d) labels.append(generic_label) return label_index(labels, json_labels.is_distinct)
def create_index_from_response(self, response): generic_labels = response.generic_labels labels = [] for label_message in generic_labels.labels: fields = {} _structs.copy_struct_to_dict(label_message.fields, fields) reference_field_ids = {} _structs.copy_struct_to_dict(label_message.reference_ids, reference_field_ids) generic_label = _labels.GenericLabel( label_message.start_index, label_message.end_index, identifier=label_message.identifier, fields=fields, reference_field_ids=reference_field_ids) labels.append(generic_label) return _label_indices.presorted_label_index(labels, generic_labels.is_distinct, adapter=self)
def call_process(self, event_id, event_instance_id, params): logger.debug( 'processor_id: %s calling process on event_id: %s with event_instance_id: %s', self.processor_id, event_id, event_instance_id) p = dict(self.params or {}) if params is not None: p.update(params) with _base.Processor.enter_context() as context: request = processing_pb2.ProcessRequest( processor_id=self.processor_id, event_id=event_id, event_service_instance_id=event_instance_id) _structs.copy_dict_to_struct(p, request.params, [p]) with _base.Processor.started_stopwatch('remote_call'): try: response = self._stub.Process(request) except Exception as e: logger.error( "Error while processing event_id %s through remote pipeline " "component '%s'", event_id, self.component_id) raise e r = {} _structs.copy_struct_to_dict(response.result, r) timing_info = response.timing_info for k, v in timing_info.items(): context.add_time(k, v.ToTimedelta()) created_indices = {} for created_index in response.created_indices: try: doc_created_indices = created_indices[ created_index.document_name] except KeyError: doc_created_indices = [] created_indices[ created_index.document_name] = doc_created_indices doc_created_indices.append(created_index.index_name) return r, context.times, created_indices
def call_process(self, event_id, params): self.processed += 1 p = dict(self.params or {}) if params is not None: p.update(params) with EventProcessor.enter_context() as context: try: request = processing_pb2.ProcessRequest( processor_id=self._processor_id, event_id=event_id) _structs.copy_dict_to_struct(p, request.params, [p]) with Processor.started_stopwatch('remote_call'): response = self._stub.Process(request) r = {} _structs.copy_struct_to_dict(response.result, r) timing_info = response.timing_info for k, v in timing_info.items(): context.add_time(k, v.ToTimedelta()) created_indices = {} for created_index in response.created_indices: try: doc_created_indices = created_indices[ created_index.document_name] except KeyError: doc_created_indices = [] created_indices[ created_index.document_name] = doc_created_indices doc_created_indices.append(created_index.index_name) return r, context.times, created_indices except Exception as e: self.failure_count += 1 logger.error( 'Processor "%s" failed while processing event with id: %s', self.component_id, event_id) logger.error(e) raise e
def test_empty_struct(): struct = struct_pb2.Struct() d = {} copy_struct_to_dict(struct, d) assert d == {}
def test_str_list_field_reverse(): struct = struct_pb2.Struct() struct.get_or_create_list('bar').extend(['a', 'b', 'c']) d = {} copy_struct_to_dict(struct, d) assert d['bar'] == ['a', 'b', 'c']
def test_struct_field_reverse(): struct = struct_pb2.Struct() struct.get_or_create_struct('bar')['bar'] = 'baz' d = {} copy_struct_to_dict(struct, d) assert d['bar'] == {'bar': 'baz'}
def test_null_field_reverse(): struct = struct_pb2.Struct() struct['bar'] = None d = {} copy_struct_to_dict(struct, d) assert d == {'bar': None}
def test_bool_field_reverse(): struct = struct_pb2.Struct() struct['bar'] = True d = {} copy_struct_to_dict(struct, d) assert d['bar'] is True
def test_int_field_reverse(): struct = struct_pb2.Struct() struct['bar'] = 1 d = {} copy_struct_to_dict(struct, d) assert d['bar'] == 1
def test_int_list_field_reverse(): struct = struct_pb2.Struct() struct.get_or_create_list('bar').extend([1, 2, 3]) d = {} copy_struct_to_dict(struct, d) assert d['bar'] == [1, 2, 3]
def test_bool_list_field_reverse(): struct = struct_pb2.Struct() struct.get_or_create_list('bar').extend([True, False, True]) d = {} copy_struct_to_dict(struct, d) assert d['bar'] == [True, False, True]
def test_empty_list_field_reverse(): struct = struct_pb2.Struct() struct.get_or_create_list('bar') d = {} copy_struct_to_dict(struct, d) assert d['bar'] == []