def _check_msg_is_not_error(msg):
if msg.error():
LOG.error("Consumer error: %s", msg.error())
return None
LOG.debug("Message from topic: %s", msg.value().decode('utf-8'))
return msg
def add_versions_from_json_file(self, version_file_location):
"""
Load a json file from local storage and append these versions to the
version tracking dictionary
Args:
version_file_location (str): Path to the json file
Returns: None
"""
try:
with open(version_file_location, "r") as content:
file_content = content.read()
version_dict_from_file = json.loads(file_content)
self.add_dictionary_to_versions(version_dict_from_file)
except FileNotFoundError as fnf:
log.error("No versioning file found at : %s",
version_file_location)
raise fnf
except json.decoder.JSONDecodeError as decode_error:
log.error("JSON file failed to decode, check version dict is "
"correctly formatted")
raise decode_error
except Exception as exception:
log.error("unknown error")
log.error(traceback.format_exc())
raise exception
def instantiate_producer(self):
"""
Try to connect to the Kafka bootstrap server. We include
functionality to allow failure to connect to the queue to happen
silently
Returns (Producer): Kafka Producer
"""
try:
LOG.debug("Instantiating Producer")
self.producer = create_producer(self.producer_config)
# Check the connection works by polling for messages
LOG.debug("Polling Queue")
self.producer.poll(3)
LOG.info("Succesfully polled Kafka Queue")
except Exception as exception:
self.producer = None
LOG.error("Kafka Producer failed to instantiate: \n %s", exception)
if self.raise_exception_on_failed_connection:
raise NoProducerInstantiatedError()
def _validate_field_names(field_names):
for field_name in field_names:
# check for strings which only contains letters, numbers and underscores
if not re.match("^[A-Za-z0-9_]+$", field_name):
LOG.error("Unsupported field name: %s", field_name)
raise Exception("Unsupported field name")
def _validate_field_names_and_types_count(field_names, field_types):
if len(field_names) != len(field_types):
LOG.error("Number of field names and number of field types are not matching")
raise Exception("Field names and types are not matching")