from agent.modules.logger import get_logger
from agent.pipeline.config import stages
from agent.pipeline.config.handlers.base import NoSchemaConfigHandler, SchemaConfigHandler, TestConfigHandler
logger = get_logger(__name__)
class PromQLConfigHandler(NoSchemaConfigHandler):
stages_to_override = {
'source': stages.source.promql.PromQLScript,
'ExpressionEvaluator_03': stages.expression_evaluator.Filtering,
'JythonEvaluator_01': stages.jython.PromQLCreateMetrics,
'ExpressionEvaluator_02': stages.expression_evaluator.AddProperties,
'JythonEvaluator_02': stages.jython.ReplaceIllegalChars,
'destination': stages.destination.Destination
}
class PromQLSchemaConfigHandler(SchemaConfigHandler):
stages_to_override = {
'source':
stages.source.promql.PromQLScript,
'ExpressionEvaluator_03':
stages.expression_evaluator.Filtering,
'JavaScriptEvaluator_01':
stages.js_convert_metrics.JSConvertMetrics30,
'ExpressionEvaluator_02':
stages.expression_evaluator.AddProperties,
'JythonEvaluator_01':
stages.jython.ReplaceIllegalChars,
'destination':
import anodot
import argparse
import gzip
import traceback
from agent import destination, monitoring
from agent.modules import proxy, logger
from kafka import KafkaConsumer
logger_ = logger.get_logger('scripts.kafka_topology.run', stdout=True)
def get_file_path(name):
return f'/tmp/{name}'
def read_data(topic, file_type, brokers: list) -> int:
consumer = KafkaConsumer(topic,
group_id='anodot_topology',
bootstrap_servers=brokers,
value_deserializer=lambda m: m.decode('utf-8'),
consumer_timeout_ms=5000,
auto_offset_reset='earliest',
enable_auto_commit=False)
count_messages = 0
with open(get_file_path(file_type), 'w') as f:
for msg in consumer:
f.write(msg.value)
f.write('\n')
count_messages += 1
import sdc_client
from agent import pipeline, streamsets
from agent.modules.logger import get_logger
from agent.streamsets import StreamSets
logger = get_logger(__name__, stdout=True)
def create_streamsets(streamsets_: StreamSets):
streamsets.repository.save(streamsets_)
def delete_streamsets(streamsets_: StreamSets):
if _has_pipelines(streamsets_):
if not _can_move_pipelines():
raise StreamsetsException(
'Cannot move pipelines to a different streamsets as only one streamsets instance exists, cannot delete streamsets that has pipelines'
)
sdc_client.StreamsetsBalancer().unload_streamsets(streamsets_)
streamsets.repository.delete(streamsets_)
def _has_pipelines(streamsets_: StreamSets) -> bool:
return len(pipeline.repository.get_by_streamsets_id(streamsets_.id)) > 0
def _can_move_pipelines():
return len(streamsets.repository.get_all()) > 1
import prometheus_client
import sdc_client
import re
from typing import List
from agent import streamsets, pipeline, source
from agent.monitoring import metrics
from agent.modules import constants
from agent.modules import logger
logger_ = logger.get_logger(__name__, stdout=True)
def _pull_system_metrics(streamsets_: streamsets.StreamSets, jmx: dict):
for bean in jmx['beans']:
if bean['name'] == 'java.lang:type=Memory':
metrics.STREAMSETS_HEAP_MEMORY.labels(streamsets_.url).set(bean['HeapMemoryUsage']['used'])
metrics.STREAMSETS_NON_HEAP_MEMORY.labels(streamsets_.url).set(bean['NonHeapMemoryUsage']['used'])
elif bean['name'] == 'java.lang:type=OperatingSystem':
metrics.STREAMSETS_CPU.labels(streamsets_.url).set(bean['ProcessCpuLoad'])
def _increase_counter(total: int, metric: prometheus_client.Counter):
# TODO: do not access private property
val = total - metric._value.get()
if val > 0:
metric.inc(val)
def _is_influx(pipeline_: pipeline.Pipeline):
import time
from agent import monitoring
from agent.modules import logger
logger_ = logger.get_logger(__name__)
SCRIPT_NAME = 'monitoring'
start = time.time()
if __name__ == '__main__':
try:
monitoring.run()
monitoring.set_scheduled_script_execution_time(SCRIPT_NAME,
time.time() - start)
except Exception as e:
logger_.error(str(e))
monitoring.increase_scheduled_script_error_counter(SCRIPT_NAME)
raise
def config(binder):
binder.install(source.config)
binder.install(pipeline.config)
binder.install(streamsets.config)
binder.bind(sdc_client.ILogger, get_logger('sdc_client', stdout=True))
import argparse
import json
import os
from agent.modules import logger
from agent import source, pipeline, destination, streamsets
logger_ = logger.get_logger('scripts.migrate-to-db.run', stdout=True)
def run(data_dir):
if len(streamsets.repository.get_all()) == 0:
print(
'You haven\'t created a streamsets instance in the database, please run `agent streamsets add` first'
)
exit(1)
migrate_destination(data_dir)
migrate_sources(data_dir)
migrate_pipelines(data_dir)
def migrate_destination(data_dir):
dest_path = os.path.join(data_dir, 'destination.json')
if not os.path.isfile(dest_path):
raise Exception('Destination does not exist, cannot migrate')
with open(dest_path) as f:
if destination.repository.exists():
logger_.info('Destination already exists, skipping')
return
data = json.load(f)
dest = destination.HttpDestination()
from copy import deepcopy
from agent import pipeline, source, di
from agent.modules import logger
logger_ = logger.get_logger('scripts.upgrade.3.13.0', stdout=True)
di.init()
for pipeline_ in pipeline.repository.get_by_type(source.TYPE_INFLUX):
logger_.info(f'Updating `{pipeline_.name}` pipeline')
values = {}
config = deepcopy(pipeline_.config)
for value in config['value']['values']:
values[value] = config.get("target_type", "gauge")
config['values'] = values
del config['value']
pipeline_.set_config(config)
pipeline.manager.update(pipeline_)
pipeline.repository.save(pipeline_)
logger_.info('Done')
logger_.info('Finished influx pipelines update')
