def configure_exchange(properties_file_path, project_path=False):
"""Read properties file, return send message function.
Args:
properties_file (str): Path of the ``application.properties`` file
containing spring cloud stream configuration.
project_path (bool): If True, treat the directory as a spring boot
project root directory. Defaults to False.
Returns:
Partial that takes a message body as an argument and sends the given
message body to the exchange configured via the read parameters.
"""
if project_path:
properties_file_path = os.path.join(properties_file_path,
"src/main/resources")
with open(os.path.join(properties_file_path, "application.properties"),
"rb") as f:
properties_dict = javaproperties.load(f)
properties_address, properties_port = properties_dict[
"spring.rabbitmq.addresses"].split(",")[0].split(":")
properties_exchange = properties_dict[
"spring.cloud.stream.bindings.input.destination"]
return functools.partial(send_message,
rabbit_address=properties_address,
rabbit_port=int(properties_port),
to_exchange=properties_exchange)
def __read_kv_from_file(file_path, format_, separator=None, prefix_to_add="", depth=None):
config_data = {}
try:
with io.open(file_path, 'r', encoding=__check_file_encoding(file_path)) as config_file:
if format_ == 'json':
config_data = json.load(config_file)
elif format_ == 'yaml':
for yaml_data in list(yaml.load_all(config_file)):
config_data.update(yaml_data)
elif format_ == 'properties':
config_data = javaproperties.load(config_file)
except ValueError:
raise CLIError(
'The input is not a well formatted %s file.' % (format_))
except OSError:
raise CLIError('File is not available.')
flattened_data = {}
index = 0
is_list = isinstance(config_data, list)
for key in config_data:
if is_list:
__flatten_key_value(prefix_to_add + str(index), key, flattened_data,
sys.maxsize if depth is None else int(depth), separator)
index += 1
else:
__flatten_key_value(
prefix_to_add + key, config_data[key], flattened_data, sys.maxsize if depth is None else int(depth), separator)
# convert to KeyValue list
key_values = []
for k, v in flattened_data.items():
key_values.append(KeyValue(key=k, value=v))
return key_values
def get_maven_pom(location=None, text=None, check_is_pom=False, extra_properties=None):
"""
Return a MavenPom object from a POM file at `location` or provided as a
`text` string.
"""
if location and check_is_pom and not is_pom(location):
return
pom = MavenPom(location, text)
if not extra_properties:
extra_properties = {}
# do we have a pom.properties file side-by-side?
if location and os.path.exists(location):
parent = fileutils.parent_directory(location)
pom_properties = os.path.join(parent, 'pom.properties')
if os.path.exists(pom_properties):
with open(pom_properties) as props:
properties = javaproperties.load(props) or {}
if TRACE:
logger.debug('_get_mavenpom: properties: {}'.format(repr(properties)))
extra_properties.update(properties)
pom.resolve(**extra_properties)
# TODO: we cannot do much without these??
if check_is_pom and not has_basic_pom_attributes(pom):
if TRACE:
logger.debug('_get_mavenpom: has_basic_pom_attributes: {}'.format(
has_basic_pom_attributes(pom)))
return
return pom
def get_maven_pom(location=None):
"""
Return a MavenPom object from a POM file at `location` or provided as a
`text` string.
"""
pom = MavenPom(location=location)
extra_properties = {}
# do we have a pom.properties file side-by-side?
# FIXME: we should treat pom.properties as a datafile
if location and os.path.exists(location):
parent = fileutils.parent_directory(location)
pom_properties = os.path.join(parent, 'pom.properties')
if os.path.exists(pom_properties):
with open(pom_properties) as props:
properties = javaproperties.load(props) or {}
if TRACE:
logger.debug(f'get_maven_pom: properties: {properties!r}')
extra_properties.update(properties)
pom.resolve(**extra_properties)
# TODO: we cannot do much without these??
hbpa = has_basic_pom_attributes(pom)
if not hbpa:
if TRACE:
logger.debug(f'get_maven_pom: has_basic_pom_attributes: {hbpa}')
return
return pom
def __read_kv_from_file(file_path,
format_,
separator=None,
prefix_to_add="",
depth=None):
config_data = {}
try:
with io.open(file_path, 'r',
encoding=__check_file_encoding(file_path)) as config_file:
if format_ == 'json':
config_data = json.load(config_file)
for feature_management_keyword in FEATURE_MANAGEMENT_KEYWORDS:
# delete all feature management sections in any name format.
# If users have not skipped features, and there are multiple
# feature sections, we will error out while reading features.
if feature_management_keyword in config_data:
del config_data[feature_management_keyword]
elif format_ == 'yaml':
for yaml_data in list(yaml.safe_load_all(config_file)):
config_data.update(yaml_data)
for feature_management_keyword in FEATURE_MANAGEMENT_KEYWORDS:
# delete all feature management sections in any name format.
# If users have not skipped features, and there are multiple
# feature sections, we will error out while reading features.
if feature_management_keyword in config_data:
del config_data[feature_management_keyword]
elif format_ == 'properties':
config_data = javaproperties.load(config_file)
logger.debug(
"Importing feature flags from a properties file is not supported. If properties file contains feature flags, they will be imported as regular key-values."
)
except ValueError:
raise CLIError('The input is not a well formatted %s file.' %
(format_))
except OSError:
raise CLIError('File is not available.')
flattened_data = {}
index = 0
is_list = isinstance(config_data, list)
for key in config_data:
if is_list:
__flatten_key_value(prefix_to_add + str(index), key,
flattened_data,
sys.maxsize if depth is None else int(depth),
separator)
index += 1
else:
__flatten_key_value(prefix_to_add + key, config_data[key],
flattened_data,
sys.maxsize if depth is None else int(depth),
separator)
# convert to KeyValue list
key_values = []
for k, v in flattened_data.items():
key_values.append(KeyValue(key=k, value=v))
return key_values
def properties2json(infile, outfile, encoding):
"""Convert a Java .properties file to JSON"""
with click.open_file(infile, encoding=encoding) as fp:
props = load(fp)
with click.open_file(outfile, 'w') as fp:
json.dump(props, fp, sort_keys=True, indent=4, separators=(',', ': '))
fp.write('\n')
def modify_conf_properties(filename, modified_properties):
with open(filename, 'r') as fp:
properties = javaproperties.load(fp)
with open(filename, 'w') as fp:
properties.update(modified_properties)
javaproperties.dump(properties, fp)
return properties
def get_repo_path(path):
# Checking input.properties file
with open('input.properties', 'rb') as f:
p = javaproperties.load(f)
if p['repo_path'] == ("\""+path+"\""):
return True
else:
return False
def set(self, key, value):
if os.path.exists(self.__filename):
with open(self.__filename, 'r') as fread:
result = javaproperties.load(fread)
result[key] = value
os.remove(self.__filename)
with open(self.__filename, 'w') as fwrite:
javaproperties.dump(result, fwrite, timestamp=None)
return False
def getproperties(fp, keys):
keys = set(keys)
def getprops(seq):
props = {}
for k,v in seq:
if k in keys:
props[k] = v
return props
return load(fp, object_pairs_hook=getprops)
def change_properties_value(self, prop_file_path, prop_key, value):
"""
Changes the value of the parameter ``prop_key`` inside the properties file
located at ``prop_file_path`` to the given ``value``
"""
prop_file = io.open(prop_file_path, "r", encoding="utf-8")
properties = javaproperties.load(prop_file)
properties[prop_key] = str(value)
with io.open(prop_file_path, "w", encoding="utf-8") as fp:
javaproperties.dump(properties, fp, sort_keys=True)
def _load_specific_env_properties(env, prop_prefix=None):
"""
Loads the env.properties file that corresponds with the given *env* str.
:param str env: A valid environment, such as: dev, test, stag, prod
:param str prop_prefix: A key prefix to filter by, like: 'api-status.'
:return: dict containing the properties from the specified env file
:rtype: dict
"""
filename = '/etc/catalist/%s/env.properties' % env.strip().lower()
prop_file = open(filename)
return javaproperties.load(
prop_file, object_pairs_hook=_filter_generator(prop_prefix))
def load(file: IO[str]) -> Report:
"""
Loads a report file from an IO stream.
:param file: The stream to read the report from.
:return: The report.
"""
# Load the report file as a Java properties file
properties: Dict[str, Any] = javaproperties.load(file)
# Parse the properties into a report
return from_properties(properties)
def __read_with_appropriate_encoding(file_path, format_):
config_data = {}
default_encoding = 'utf-8'
detected_encoding = __check_file_encoding(file_path)
try:
with io.open(file_path, 'r', encoding=default_encoding) as config_file:
if format_ == 'json':
config_data = json.load(config_file)
elif format_ == 'yaml':
for yaml_data in list(yaml.safe_load_all(config_file)):
config_data.update(yaml_data)
elif format_ == 'properties':
config_data = javaproperties.load(config_file)
logger.debug(
"Importing feature flags from a properties file is not supported. If properties file contains feature flags, they will be imported as regular key-values."
)
except (UnicodeDecodeError, json.JSONDecodeError):
if detected_encoding == default_encoding:
raise
with io.open(file_path, 'r',
encoding=detected_encoding) as config_file:
if format_ == 'json':
config_data = json.load(config_file)
elif format_ == 'yaml':
for yaml_data in list(yaml.safe_load_all(config_file)):
config_data.update(yaml_data)
elif format_ == 'properties':
config_data = javaproperties.load(config_file)
logger.debug(
"Importing feature flags from a properties file is not supported. If properties file contains feature flags, they will be imported as regular key-values."
)
return config_data
def __read_kv_from_file(file_path,
format_,
separator=None,
prefix_to_add="",
depth=None):
config_data = {}
try:
with io.open(file_path, 'r',
encoding=__check_file_encoding(file_path)) as config_file:
if format_ == 'json':
config_data = json.load(config_file)
if 'FeatureManagement' in config_data:
del config_data['FeatureManagement']
elif format_ == 'yaml':
for yaml_data in list(yaml.safe_load_all(config_file)):
config_data.update(yaml_data)
logger.warning(
"Importing feature flags from a yaml file is not supported yet. If yaml file contains feature flags, they will be imported as regular key-values."
)
elif format_ == 'properties':
config_data = javaproperties.load(config_file)
logger.warning(
"Importing feature flags from a properties file is not supported yet. If properties file contains feature flags, they will be imported as regular key-values."
)
except ValueError:
raise CLIError('The input is not a well formatted %s file.' %
(format_))
except OSError:
raise CLIError('File is not available.')
flattened_data = {}
index = 0
is_list = isinstance(config_data, list)
for key in config_data:
if is_list:
__flatten_key_value(prefix_to_add + str(index), key,
flattened_data,
sys.maxsize if depth is None else int(depth),
separator)
index += 1
else:
__flatten_key_value(prefix_to_add + key, config_data[key],
flattened_data,
sys.maxsize if depth is None else int(depth),
separator)
# convert to KeyValue list
key_values = []
for k, v in flattened_data.items():
key_values.append(KeyValue(key=k, value=v))
return key_values
def read_local_config(config_file):
if not os.path.exists(config_file):
raise KafkaStreamsError(f'Config file {config_file} does not exist')
with open(config_file, 'r') as cf:
props = javaproperties.load(cf)
for k, v in props.items():
ku = k.upper().replace('.', '_')
if ku not in globals().keys():
raise KafkaStreamsError(f'Unrecognised property {k} read from config file {config_file}')
globals()[ku] = v
log.debug('Config from "%s": %s = %s', config_file, k, v)
def properties2json(infile, outfile, encoding, ensure_ascii, sort_keys):
"""Convert a Java .properties file to JSON"""
with click.open_file(infile, encoding=encoding) as fp:
props = load(fp, object_pairs_hook=OrderedDict)
with click.open_file(outfile, "w", encoding="utf-8") as fp:
json.dump(
props,
fp,
sort_keys=sort_keys,
indent=4,
separators=(",", ": "),
ensure_ascii=ensure_ascii,
)
fp.write("\n")
def get_properties_file_content(self, prop_file_path):
"""
Reads the complete content of a properties file located at
``prop_file_path`` and returns its content as a dictionary
"""
try:
prop_file = open(prop_file_path, "r")
properties = javaproperties.load(prop_file)
prop_file.close()
return properties
except OSError as err:
print("File cannot be opened: " + err)
def format(outfile, separator, file, encoding, ensure_ascii):
"""Format/"canonicalize" a Java .properties file"""
with click.open_file(file, encoding=encoding) as fpin:
with click.open_file(
outfile,
"w",
encoding=encoding,
errors="javapropertiesreplace",
) as fpout:
dump(
load(fpin),
fpout,
sort_keys=True,
separator=separator,
ensure_ascii=ensure_ascii,
)
def parse(cls, location):
"""
Yield PackageData from a pom.properties file (which is typically side-
by-side with its pom file.)
"""
with open(location) as props:
properties = javaproperties.load(props) or {}
if TRACE:
logger.debug(
f'MavenPomPropertiesHandler.parse: properties: {properties!r}'
)
if properties:
yield models.PackageData(
datasource_id=cls.datasource_id,
type=cls.package_type,
primary_language=cls.primary_language,
extra_data=dict(pom_properties=properties))
def get_directories_in_properties(plugin_path):
build_properties_path = os.path.join(plugin_path, 'build.properties')
if not os.path.isfile(build_properties_path):
raise InvalidPluginException(
'missing build.properties in the plugin %s' %
red(plugin_name(plugin_path)))
build_properties = javaproperties.load(open(build_properties_path, 'r'))
source_directory_in_properties = build_properties[
'source..'] if 'source..' in build_properties else None
output_directory_in_properties = build_properties[
'output..'] if 'output..' in build_properties else None
if source_directory_in_properties is not None and source_directory_in_properties.endswith(
'/'):
source_directory_in_properties = source_directory_in_properties[:-1]
if output_directory_in_properties is not None and output_directory_in_properties.endswith(
'/'):
output_directory_in_properties = output_directory_in_properties[:-1]
return source_directory_in_properties, output_directory_in_properties
def main():
signal.signal(signal.SIGTERM, signal_handler)
parser = argparse.ArgumentParser(
description='Export Kafka consumer offsets to Prometheus.')
parser.add_argument(
'-b',
'--bootstrap-brokers',
help='Addresses of brokers in a Kafka cluster to talk to.' +
' Brokers should be separated by commas e.g. broker1,broker2.' +
' Ports can be provided if non-standard (9092) e.g. brokers1:9999.' +
' (default: localhost)')
parser.add_argument(
'-p',
'--port',
type=int,
default=9208,
help='Port to serve the metrics endpoint on. (default: 9208)')
parser.add_argument(
'-s',
'--from-start',
action='store_true',
help='Start from the beginning of the `__consumer_offsets` topic.')
parser.add_argument(
'--topic-interval',
type=float,
default=30.0,
help='How often to refresh topic information, in seconds. (default: 30)'
)
parser.add_argument(
'--high-water-interval',
type=float,
default=10.0,
help=
'How often to refresh high-water information, in seconds. (default: 10)'
)
parser.add_argument(
'--low-water-interval',
type=float,
default=10.0,
help=
'How often to refresh low-water information, in seconds. (default: 10)'
)
parser.add_argument(
'--consumer-config',
action='append',
default=[],
help=
'Provide additional Kafka consumer config as a consumer.properties file. Multiple files will be merged, later files having precedence.'
)
parser.add_argument('-j',
'--json-logging',
action='store_true',
help='Turn on json logging.')
parser.add_argument(
'--log-level',
default='INFO',
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help='Detail level to log. (default: INFO)')
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='Turn on verbose (DEBUG) logging. Overrides --log-level.')
args = parser.parse_args()
log_handler = logging.StreamHandler()
log_format = '[%(asctime)s] %(name)s.%(levelname)s %(threadName)s %(message)s'
formatter = JogFormatter(log_format) \
if args.json_logging \
else logging.Formatter(log_format)
log_handler.setFormatter(formatter)
log_level = getattr(logging, args.log_level)
logging.basicConfig(handlers=[log_handler],
level=logging.DEBUG if args.verbose else log_level)
logging.captureWarnings(True)
port = args.port
consumer_config = {
'bootstrap_servers': 'localhost',
'auto_offset_reset': 'latest',
'group_id': None,
'consumer_timeout_ms': 500
}
args.consumer_config.append(os.environ.get('CONSUMER_CONFIG'))
for filename in args.consumer_config:
with open(filename) as f:
raw_config = javaproperties.load(f)
for k, v in raw_config.items():
if v == '':
# Treat empty values as if they weren't set
continue
if v.lower() in ['true', 'false']:
# Convert boolean values
v = True if v.lower() == 'true' else False
else:
# Try and convert numeric values
try:
v = int(v)
except ValueError:
try:
v = float(v)
except ValueError:
pass
consumer_config[k.replace('.', '_')] = v
if args.bootstrap_brokers:
consumer_config['bootstrap_servers'] = args.bootstrap_brokers
consumer_config['bootstrap_servers'] = consumer_config[
'bootstrap_servers'].split(',')
if args.from_start:
consumer_config['auto_offset_reset'] = 'earliest'
consumer = KafkaConsumer('__consumer_offsets', **consumer_config)
client = consumer._client
topic_interval = args.topic_interval
high_water_interval = args.high_water_interval
low_water_interval = args.low_water_interval
logging.info('Starting server...')
start_http_server(port)
logging.info('Server started on port %s', port)
REGISTRY.register(collectors.HighwaterCollector())
REGISTRY.register(collectors.LowwaterCollector())
REGISTRY.register(collectors.ConsumerOffsetCollector())
REGISTRY.register(collectors.ConsumerLagCollector())
REGISTRY.register(collectors.ConsumerLeadCollector())
REGISTRY.register(collectors.ConsumerCommitsCollector())
REGISTRY.register(collectors.ConsumerCommitTimestampCollector())
REGISTRY.register(collectors.ExporterOffsetCollector())
REGISTRY.register(collectors.ExporterLagCollector())
REGISTRY.register(collectors.ExporterLeadCollector())
scheduled_jobs = setup_fetch_jobs(topic_interval, high_water_interval,
low_water_interval, client)
scheduler.run_scheduled_jobs(scheduled_jobs)
try:
while True:
for message in consumer:
offsets = collectors.get_offsets()
commits = collectors.get_commits()
commit_timestamps = collectors.get_commit_timestamps()
exporter_offsets = collectors.get_exporter_offsets()
# Commits store the offset a consumer should read from next,
# so we need to add one to the current offset for semantic parity
exporter_partition = message.partition
exporter_offset = message.offset + 1
exporter_offsets = ensure_dict_key(exporter_offsets,
exporter_partition,
exporter_offset)
exporter_offsets[exporter_partition] = exporter_offset
collectors.set_exporter_offsets(exporter_offsets)
if message.key:
key_dict = parse_key(message.key)
# Only key versions 0 and 1 are offset commit messages.
# Ignore other versions.
if key_dict is not None and key_dict['version'] in (0, 1):
if message.value:
value_dict = parse_value(message.value)
if value_dict is not None:
group = key_dict['group']
topic = key_dict['topic']
partition = key_dict['partition']
offset = value_dict['offset']
commit_timestamp = value_dict[
'commit_timestamp'] / 1000
offsets = ensure_dict_key(offsets, group, {})
offsets[group] = ensure_dict_key(
offsets[group], topic, {})
offsets[group][topic] = ensure_dict_key(
offsets[group][topic], partition, offset)
offsets[group][topic][partition] = offset
collectors.set_offsets(offsets)
commits = ensure_dict_key(commits, group, {})
commits[group] = ensure_dict_key(
commits[group], topic, {})
commits[group][topic] = ensure_dict_key(
commits[group][topic], partition, 0)
commits[group][topic][partition] += 1
collectors.set_commits(commits)
commit_timestamps = ensure_dict_key(
commit_timestamps, group, {})
commit_timestamps[group] = ensure_dict_key(
commit_timestamps[group], topic, {})
commit_timestamps[group][
topic] = ensure_dict_key(
commit_timestamps[group][topic],
partition, 0)
commit_timestamps[group][topic][
partition] = commit_timestamp
collectors.set_commit_timestamps(
commit_timestamps)
else:
# The group has been removed, so we should not report metrics
group = key_dict['group']
topic = key_dict['topic']
partition = key_dict['partition']
if group in offsets:
if topic in offsets[group]:
if partition in offsets[group][topic]:
del offsets[group][topic][partition]
if group in commits:
if topic in commits[group]:
if partition in commits[group][topic]:
del commits[group][topic][partition]
if group in commit_timestamps:
if topic in commit_timestamps[group]:
if partition in commit_timestamps[group][
topic]:
del commit_timestamps[group][topic][
partition]
# Check if we need to run any scheduled jobs
# each message.
scheduled_jobs = scheduler.run_scheduled_jobs(scheduled_jobs)
# Also check if we need to run any scheduled jobs
# each time the consumer times out, in case there
# aren't any messages to consume.
scheduled_jobs = scheduler.run_scheduled_jobs(scheduled_jobs)
except KeyboardInterrupt:
pass
shutdown()
def __init__(self):
self._filename = "settings.properties"
f = open(self._filename, 'r')
self._datalist = javaproperties.load(f)
def format(outfile, separator, file, encoding):
""" Format/"canonicalize" a Java .properties file """
with click.open_file(file, encoding=encoding) as fpin:
with click.open_file(outfile, 'w', encoding=encoding) as fpout:
dump(load(fpin), fpout, sort_keys=True, separator=separator)
def properties_language_file_to_dict(plftd_file_path):
with open(plftd_file_path, "r", encoding="utf-8",
errors="replace") as plftd_f:
plftd_dict = javaproperties.load(plftd_f)
return plftd_dict
def get(self, key):
if os.path.exists(self.__filename):
with open(self.__filename, 'r') as fread:
result = javaproperties.load(fread)
return result.get(key)
return False
def main():
signal.signal(signal.SIGTERM, signal_handler)
parser = argparse.ArgumentParser(
description='Export Kafka consumer offsets to Prometheus.')
parser.add_argument(
'-b',
'--bootstrap-brokers',
default='localhost',
help='Addresses of brokers in a Kafka cluster to talk to.' +
' Brokers should be separated by commas e.g. broker1,broker2.' +
' Ports can be provided if non-standard (9092) e.g. brokers1:9999.' +
' (default: localhost)')
parser.add_argument(
'-p',
'--port',
type=int,
default=9208,
help='Port to serve the metrics endpoint on. (default: 9208)')
parser.add_argument(
'-s',
'--from-start',
action='store_true',
help='Start from the beginning of the `__consumer_offsets` topic.')
parser.add_argument(
'--topic-interval',
type=float,
default=30.0,
help='How often to refresh topic information, in seconds. (default: 30)'
)
parser.add_argument(
'--high-water-interval',
type=float,
default=10.0,
help=
'How often to refresh high-water information, in seconds. (default: 10)'
)
parser.add_argument(
'--consumer-config',
action='append',
default=[],
help=
'Provide additional Kafka consumer config as a consumer.properties file. Multiple files will be merged, later files having precedence.'
)
parser.add_argument('-j',
'--json-logging',
action='store_true',
help='Turn on json logging.')
parser.add_argument(
'--log-level',
default='INFO',
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help='detail level to log. (default: INFO)')
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='turn on verbose (DEBUG) logging. Overrides --log-level.')
args = parser.parse_args()
log_handler = logging.StreamHandler()
log_format = '[%(asctime)s] %(name)s.%(levelname)s %(threadName)s %(message)s'
formatter = JogFormatter(log_format) \
if args.json_logging \
else logging.Formatter(log_format)
log_handler.setFormatter(formatter)
log_level = getattr(logging, args.log_level)
logging.basicConfig(handlers=[log_handler],
level=logging.DEBUG if args.verbose else log_level)
logging.captureWarnings(True)
port = args.port
consumer_config = {
'bootstrap_servers': 'localhost',
'auto_offset_reset': 'latest',
'group_id': None,
'consumer_timeout_ms': 500
}
for filename in args.consumer_config:
with open(filename) as f:
raw_config = javaproperties.load(f)
converted_config = {
k.replace('.', '_'): v
for k, v in raw_config.items()
}
consumer_config.update(converted_config)
if args.bootstrap_brokers:
consumer_config['bootstrap_servers'] = args.bootstrap_brokers.split(
',')
if args.from_start:
consumer_config['auto_offset_reset'] = 'earliest'
consumer = KafkaConsumer('__consumer_offsets', **consumer_config)
client = consumer._client
topic_interval = args.topic_interval
high_water_interval = args.high_water_interval
logging.info('Starting server...')
start_http_server(port)
logging.info('Server started on port %s', port)
def read_short(bytes):
num = unpack_from('>h', bytes)[0]
remaining = bytes[2:]
return (num, remaining)
def read_int(bytes):
num = unpack_from('>i', bytes)[0]
remaining = bytes[4:]
return (num, remaining)
def read_long_long(bytes):
num = unpack_from('>q', bytes)[0]
remaining = bytes[8:]
return (num, remaining)
def read_string(bytes):
length, remaining = read_short(bytes)
string = remaining[:length].decode('utf-8')
remaining = remaining[length:]
return (string, remaining)
def parse_key(bytes):
(version, remaining_key) = read_short(bytes)
if version == 1 or version == 0:
(group, remaining_key) = read_string(remaining_key)
(topic, remaining_key) = read_string(remaining_key)
(partition, remaining_key) = read_int(remaining_key)
return (version, group, topic, partition)
def parse_value(bytes):
(version, remaining_key) = read_short(bytes)
if version == 0:
(offset, remaining_key) = read_long_long(remaining_key)
(metadata, remaining_key) = read_string(remaining_key)
(timestamp, remaining_key) = read_long_long(remaining_key)
return (version, offset, metadata, timestamp)
elif version == 1:
(offset, remaining_key) = read_long_long(remaining_key)
(metadata, remaining_key) = read_string(remaining_key)
(commit_timestamp, remaining_key) = read_long_long(remaining_key)
(expire_timestamp, remaining_key) = read_long_long(remaining_key)
return (version, offset, metadata, commit_timestamp,
expire_timestamp)
def update_topics(api_version, metadata):
logging.info('Received topics and partition assignments')
global topics
if api_version == 0:
TOPIC_ERROR = 0
TOPIC_NAME = 1
TOPIC_PARTITIONS = 2
PARTITION_ERROR = 0
PARTITION_NUMBER = 1
PARTITION_LEADER = 2
else:
TOPIC_ERROR = 0
TOPIC_NAME = 1
TOPIC_PARTITIONS = 3
PARTITION_ERROR = 0
PARTITION_NUMBER = 1
PARTITION_LEADER = 2
new_topics = {}
for t in metadata.topics:
error_code = t[TOPIC_ERROR]
if error_code:
error = Errors.for_code(error_code)(t)
logging.warning(
'Received error in metadata response at topic level: %s',
error)
else:
topic = t[TOPIC_NAME]
partitions = t[TOPIC_PARTITIONS]
new_partitions = {}
for p in partitions:
error_code = p[PARTITION_ERROR]
if error_code:
error = Errors.for_code(error_code)(p)
logging.warning(
'Received error in metadata response at partition level for topic %(topic)s: %(error)s',
{
'topic': topic,
'error': error
})
else:
partition = p[PARTITION_NUMBER]
leader = p[PARTITION_LEADER]
logging.debug(
'Received partition assignment for partition %(partition)s of topic %(topic)s',
{
'partition': partition,
'topic': topic
})
new_partitions[partition] = leader
new_topics[topic] = new_partitions
topics = new_topics
def update_highwater(offsets):
logging.info('Received high-water marks')
for topic, partitions in offsets.topics:
for partition, error_code, offsets in partitions:
if error_code:
error = Errors.for_code(error_code)(
(partition, error_code, offsets))
logging.warning(
'Received error in offset response for topic %(topic)s: %(error)s',
{
'topic': topic,
'error': error
})
else:
logging.debug(
'Received high-water marks for partition %(partition)s of topic %(topic)s',
{
'partition': partition,
'topic': topic
})
update_gauge(
metric_name='kafka_topic_highwater',
label_dict={
'topic': topic,
'partition': partition
},
value=offsets[0],
doc='The offset of the head of a partition in a topic.'
)
def fetch_topics(this_time):
logging.info('Requesting topics and partition assignments')
next_time = this_time + topic_interval
try:
node = client.least_loaded_node()
logging.debug(
'Requesting topics and partition assignments from %(node)s',
{'node': node})
api_version = 0 if client.config['api_version'] < (0, 10) else 1
request = MetadataRequest[api_version](None)
f = client.send(node, request)
f.add_callback(update_topics, api_version)
except Exception:
logging.exception(
'Error requesting topics and partition assignments')
finally:
client.schedule(partial(fetch_topics, next_time), next_time)
def fetch_highwater(this_time):
logging.info('Requesting high-water marks')
next_time = this_time + high_water_interval
try:
global topics
if topics:
nodes = {}
for topic, partition_map in topics.items():
for partition, leader in partition_map.items():
if leader not in nodes:
nodes[leader] = {}
if topic not in nodes[leader]:
nodes[leader][topic] = []
nodes[leader][topic].append(partition)
for node, topic_map in nodes.items():
logging.debug('Requesting high-water marks from %(node)s',
{
'topic': topic,
'node': node
})
request = OffsetRequest[0](
-1,
[(topic, [(partition, OffsetResetStrategy.LATEST, 1)
for partition in partitions])
for topic, partitions in topic_map.items()])
f = client.send(node, request)
f.add_callback(update_highwater)
except Exception:
logging.exception('Error requesting high-water marks')
finally:
client.schedule(partial(fetch_highwater, next_time), next_time)
now_time = time.time()
fetch_topics(now_time)
fetch_highwater(now_time)
try:
while True:
for message in consumer:
update_gauge(
metric_name=METRIC_PREFIX + 'exporter_offset',
label_dict={'partition': message.partition},
value=message.offset,
doc=
'The current offset of the exporter consumer in a partition of the __consumer_offsets topic.'
)
if message.key and message.value:
key = parse_key(message.key)
if key:
value = parse_value(message.value)
update_gauge(
metric_name=METRIC_PREFIX + 'offset',
label_dict={
'group': key[1],
'topic': key[2],
'partition': key[3]
},
value=value[1],
doc=
'The current offset of a consumer group in a partition of a topic.'
)
increment_counter(
metric_name=METRIC_PREFIX + 'commits',
label_dict={
'group': key[1],
'topic': key[2],
'partition': key[3]
},
doc=
'The number of commit messages read by the exporter consumer from a consumer group for a partition of a topic.'
)
except KeyboardInterrupt:
pass
shutdown()
def main():
parser = argparse.ArgumentParser(
epilog="""Description:
Plays a video from a jpeg topic, visualizes the head detections and tracks, and pass detections.
Displays the result on screen ('-d') or stores result in kafka ('-o').
Required topics:
- <prefix>.cam.0.original.Image.jpg
- <prefix>.cam.0.dets.ObjectDetectionRecord.json
- <prefix>.cam.0.tracks.TrackChangeRecord.json
- <prefix>.cam.0.passdet.PassDetectionRecord.json
""",
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("broker",
help="The name of the kafka broker.",
type=str)
parser.add_argument("prefix",
help="Prefix of topics (base|skeleton).",
type=str)
parser.add_argument("config", help="Path to service config.", type=str)
parser.add_argument('-f', "--full_screen", action='store_true')
parser.add_argument('-d', "--display", action='store_true')
parser.add_argument('-v', "--video_file", action='store_true')
parser.add_argument('-o',
'--output',
help='write output image into kafka topic',
action='store_true')
args = parser.parse_args()
config_file = Path(args.config)
if not config_file.is_file():
parser.error(f"{args.config} does not exist.")
if not args.display and not args.output:
parser.error(
"Missing argument: -d (display output) or -o (write output to kafka) is needed"
)
if args.output:
producer = Producer({'bootstrap.servers': args.broker})
begin_flag = None
end_flag = None
if args.video_file:
begin_flag = BeginFlag.BEGINNING
end_flag = EndFlag.END_OF_PARTITION
with config_file.open() as f:
try:
passdet_config_json = json.loads(
javaproperties.load(f)
["ultinous.service.kafka.passdet.config"])
except KeyError:
parser.error(
"Missing property: ultinous.service.kafka.passdet.config")
except JSONDecodeError as e:
parser.error(f"Error parsing {e}")
overlay = cv2.imread('resources/powered_by_white.png',
cv2.IMREAD_UNCHANGED)
passlines: Dict[str, PassLine] = {
pl["id"]: PassLine(next(pass_colors),
[(int(p["x"]), int(p["y"])) for p in pl["poly"]])
for pl in passdet_config_json["passLines"]
}
image_topic = f"{args.prefix}.cam.0.original.Image.jpg"
detection_topic = f"{args.prefix}.cam.0.dets.ObjectDetectionRecord.json"
track_topic = f"{args.prefix}.cam.0.tracks.TrackChangeRecord.json"
frameinfo_topic = f"{args.prefix}.cam.0.frameinfo.FrameInfoRecord.json"
passdet_topic = f"{args.prefix}.cam.0.passdet.PassDetectionRecord.json"
output_topic_name = f"{args.prefix}.cam.0.passdet.Image.jpg"
# handle full screen
window_name = "DEMO: Pass detection"
if args.full_screen:
cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
cv2.setWindowProperty(window_name, cv2.WND_PROP_FULLSCREEN,
cv2.WINDOW_FULLSCREEN)
# read message, draw and display them
consumer = TimeOrderedGeneratorWithTimeout(
args.broker,
"detection", [
TopicInfo(image_topic),
TopicInfo(track_topic, drop=False),
TopicInfo(passdet_topic, drop=False),
TopicInfo(detection_topic),
TopicInfo(frameinfo_topic)
],
100,
None,
True,
begin_flag=begin_flag,
end_flag=end_flag)
i = 0
scaling = 1.0
tracks: DefaultDict[Any, ColoredPolyLine] = defaultdict(
lambda: ColoredPolyLine(next(track_colors)))
for msgs in consumer.getMessages():
for time, v in message_list_to_frame_structure(msgs).items():
for track_key, track_val in v[args.prefix]["0"]["track"].items():
if track_val["end_of_track"]:
if track_key in tracks:
del tracks[track_key]
continue
point = track_val["point"]["x"], track_val["point"]["y"]
tracks[track_key].add_point(point)
for pass_det in v[args.prefix]["0"]["passdet"].values():
if pass_det["type"] == "HEARTBEAT":
continue
elif pass_det["type"] == "END_OF_TRACK":
continue
elif pass_det["type"] == "PASS_CANDIDATE":
pass_id = pass_det["pass_candidate"]["pass"][
"pass_line_id"]
cross_dir = pass_det["pass_candidate"]["pass"]["cross_dir"]
if pass_id in passlines:
passlines[pass_id].add_event(cross_dir)
elif pass_det["type"] == "PASS_REALIZED":
continue
img = v[args.prefix]["0"]["image"]
if type(img) == np.ndarray:
# Set the image scale
shape_orig = v[args.prefix]["0"]["head_detection"].pop(
"image", {})
if shape_orig:
scaling = img.shape[1] / shape_orig["frame_info"]["columns"]
# draw bounding_box
for head_detection in v[args.prefix]["0"]["head_detection"]:
object_detection_record = v[args.prefix]["0"][
"head_detection"][head_detection]["bounding_box"]
if object_detection_record["type"] == "PERSON_HEAD":
img = draw_nice_bounding_box(
canvas=img,
bounding_box=object_detection_record[
"bounding_box"],
color=(10, 95, 255),
scaling=scaling)
for t in tracks.values():
t.draw(img, scaling)
for idx, l in enumerate(passlines.values()):
l.draw(img, scaling)
cv2.putText(img,
"".join(l.events), (40, (idx + 1) * 50),
cv2.FONT_HERSHEY_COMPLEX,
2,
l.color,
5,
bottomLeftOrigin=True)
img = draw_overlay(canvas=img,
overlay=overlay,
position=Position.BOTTOM_RIGHT,
scale=scaling)
# produce output topic
if args.output:
producer.produce(output_topic_name,
value=encode_image_to_message(img),
timestamp=time)
producer.poll(0)
if i % 100 == 0:
producer.flush()
i = 0
i += 1
# display
if args.display:
cv2.imshow(window_name, img)
k = cv2.waitKey(33)
if k == 113: # The 'q' key to stop
if args.video_file:
exit(130)
break
elif k == -1: # normally -1 returned,so don't print it
continue
else:
print(f"Press 'q' key for EXIT!")
def main():
signal.signal(signal.SIGTERM, signal_handler)
parser = argparse.ArgumentParser(
description='Export Kafka consumer offsets to Prometheus.')
parser.add_argument(
'-b',
'--bootstrap-brokers',
default='localhost',
help='Addresses of brokers in a Kafka cluster to talk to.' +
' Brokers should be separated by commas e.g. broker1,broker2.' +
' Ports can be provided if non-standard (9092) e.g. brokers1:9999.' +
' (default: localhost)')
parser.add_argument(
'-p',
'--port',
type=int,
default=9208,
help='Port to serve the metrics endpoint on. (default: 9208)')
parser.add_argument(
'-s',
'--from-start',
action='store_true',
help='Start from the beginning of the `__consumer_offsets` topic.')
parser.add_argument(
'--topic-interval',
type=float,
default=30.0,
help='How often to refresh topic information, in seconds. (default: 30)'
)
parser.add_argument(
'--high-water-interval',
type=float,
default=10.0,
help=
'How often to refresh high-water information, in seconds. (default: 10)'
)
parser.add_argument(
'--low-water-interval',
type=float,
default=10.0,
help=
'How often to refresh low-water information, in seconds. (default: 10)'
)
parser.add_argument(
'--consumer-config',
action='append',
default=[],
help=
'Provide additional Kafka consumer config as a consumer.properties file. Multiple files will be merged, later files having precedence.'
)
parser.add_argument('-j',
'--json-logging',
action='store_true',
help='Turn on json logging.')
parser.add_argument(
'--log-level',
default='INFO',
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help='detail level to log. (default: INFO)')
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='turn on verbose (DEBUG) logging. Overrides --log-level.')
args = parser.parse_args()
log_handler = logging.StreamHandler()
log_format = '[%(asctime)s] %(name)s.%(levelname)s %(threadName)s %(message)s'
formatter = JogFormatter(log_format) \
if args.json_logging \
else logging.Formatter(log_format)
log_handler.setFormatter(formatter)
log_level = getattr(logging, args.log_level)
logging.basicConfig(handlers=[log_handler],
level=logging.DEBUG if args.verbose else log_level)
logging.captureWarnings(True)
port = args.port
consumer_config = {
'bootstrap_servers': 'localhost',
'auto_offset_reset': 'latest',
'group_id': None,
'consumer_timeout_ms': 500
}
for filename in args.consumer_config:
with open(filename) as f:
raw_config = javaproperties.load(f)
converted_config = {
k.replace('.', '_'): v
for k, v in raw_config.items()
}
consumer_config.update(converted_config)
if args.bootstrap_brokers:
consumer_config['bootstrap_servers'] = args.bootstrap_brokers.split(
',')
if args.from_start:
consumer_config['auto_offset_reset'] = 'earliest'
consumer = KafkaConsumer('__consumer_offsets', **consumer_config)
client = consumer._client
topic_interval = args.topic_interval
high_water_interval = args.high_water_interval
low_water_interval = args.low_water_interval
logging.info('Starting server...')
start_http_server(port)
logging.info('Server started on port %s', port)
REGISTRY.register(collectors.HighwaterCollector())
REGISTRY.register(collectors.LowwaterCollector())
REGISTRY.register(collectors.ConsumerOffsetCollector())
REGISTRY.register(collectors.ConsumerLagCollector())
REGISTRY.register(collectors.ConsumerLeadCollector())
REGISTRY.register(collectors.ConsumerCommitsCollector())
REGISTRY.register(collectors.ExporterOffsetCollector())
REGISTRY.register(collectors.ExporterLagCollector())
REGISTRY.register(collectors.ExporterLeadCollector())
scheduled_jobs = setup_fetch_jobs(topic_interval, high_water_interval,
low_water_interval, client)
try:
while True:
for message in consumer:
offsets = collectors.get_offsets()
commits = collectors.get_commits()
exporter_offsets = collectors.get_exporter_offsets()
exporter_partition = message.partition
exporter_offset = message.offset
exporter_offsets = ensure_dict_key(exporter_offsets,
exporter_partition,
exporter_offset)
exporter_offsets[exporter_partition] = exporter_offset
collectors.set_exporter_offsets(exporter_offsets)
if message.key and message.value:
key = parse_key(message.key)
if key:
value = parse_value(message.value)
group = key[1]
topic = key[2]
partition = key[3]
offset = value[1]
offsets = ensure_dict_key(offsets, group, {})
offsets[group] = ensure_dict_key(
offsets[group], topic, {})
offsets[group][topic] = ensure_dict_key(
offsets[group][topic], partition, offset)
offsets[group][topic][partition] = offset
collectors.set_offsets(offsets)
commits = ensure_dict_key(commits, group, {})
commits[group] = ensure_dict_key(
commits[group], topic, {})
commits[group][topic] = ensure_dict_key(
commits[group][topic], partition, 0)
commits[group][topic][partition] += 1
collectors.set_commits(commits)
# Check if we need to run any scheduled jobs
# each message.
scheduled_jobs = scheduler.run_scheduled_jobs(scheduled_jobs)
# Also check if we need to run any scheduled jobs
# each time the consumer times out, in case there
# aren't any messages to consume.
scheduled_jobs = scheduler.run_scheduled_jobs(scheduled_jobs)
except KeyboardInterrupt:
pass
shutdown()
def extract_from_contexts(seed_iri,
properties,
property_f,
extract_params,
dest_dir,
verbose=False,
error=None):
'''
Loads in our seed graph from the provided IRI.
Then iterates through our provided Java property
file, which contains prefixes and IRIs for
context ontologies. Entities with the associated
prefix are pulled from the seed ontology via the seed query.
These seed entities are then used as roots for the property
crawl paths in the associated context ontology.
'''
#Read in the graph we'll pull seeds from
seed_graph = Graph().parse(seed_iri)
if verbose:
print("Loaded seed graph.")
#Connect to BioPortal
BIOPORTAL_API_KEY = os.environ['BIOPORTAL_API_KEY']
bioportal = SPARQLWrapper('http://sparql.bioontology.org/sparql/')
bioportal.addCustomParameter("apikey", BIOPORTAL_API_KEY)
print("Connected to BioPortal.")
#Create template for retrieving seed classes based on prefixes
SEED_QUERY_TEMPLATE = """
PREFIX owl: <http://www.w3.org/2002/07/owl#>
SELECT DISTINCT ?c WHERE{
?c a owl:Class .
FILTER(regex(str(?c), "%s"))
}
"""
#Read in properties file
with open(property_f, 'r') as fp:
javaprops = javaproperties.load(fp)
if verbose:
print("Read properties file.")
#Iterate through rows of properties file
for k in javaprops.keys():
if verbose:
print("==========================================")
print("Reading graph: ", str(k))
#Pull the ontology IRI and associated prefix
row = javaprops[k].split(',')
prefix = row[0]
iri = row[2]
#Check whether we have an IRI for the row
if iri == '':
if verbose:
print("No IRI provided.")
continue
if iri == seed_iri:
if verbose:
print("Context graph is same as seed graph. Skipping.")
continue
#Read in context graph from IRI, use as context
context = Graph()
FORMATS = ['xml', 'n3', 'nt', 'trix', 'turtle', 'rdfa']
read_success = False
for form in FORMATS:
try:
#If we successfully read our ontology, recurse
context = Graph().parse(iri, format=form)
read_success = True
if verbose:
print("Read as ", form, ".")
break
except Exception as e:
pass
if not read_success:
#Last-ditch effort: try to guess the file extension
try:
if verbose:
print(
"Exhausted format list. Attempting to guess format...")
context = Graph().parse(iri, format=guess_format(iri))
if verbose:
print("Read as ", guess_format(iri), ".")
break
except Exception as e:
pass
#Error handling, quiet or fast failing
if error is None:
raise Exception("Exhausted format list. Failing quickly.")
if error == 'ignore':
print("Exhausted format list. Quietly ignoring failure.")
continue
#Expand property paths from context
gout = retrieve_crawl_paths_from_context(
seed_graph=seed_graph,
context=context,
properties=properties,
seed_query=SEED_QUERY_TEMPLATE % (prefix, ),
expand_ontologies=True,
import_error=error,
verbose=verbose,
inplace=False,
extract_params=extract_params)
#Get the BioPortal extract as well
bio_seeds = {
row[0]
for row in seed_graph.query(SEED_QUERY_TEMPLATE % (prefix, ))
}
bio_gout = bioportal_retrieve_crawl_paths(
properties=properties,
bioportal=bioportal,
seeds=bio_seeds,
verbose=verbose,
extract_params=extract_params)
#Write out
gout.serialize(path.join(dest_dir, k + '.ttl'), format='turtle')
bio_gout.serialize(path.join(path.join(dest_dir, 'bioportal/'),
k + '_bioportal.ttl'),
format='turtle')
print("Wrote out extracts for " + k + ".")
del gout