def update(self, data, interval):
updated_dimensions, updated_variables = str(), str()
for dim in self.dimensions:
value = dim.get_value(data)
if value is not None:
updated_dimensions += dim.set(value)
for var in self.variables:
value = var.get_value(data)
if value is not None:
updated_variables += var.set(value)
if updated_dimensions:
since_last = interval if self.flags.updated else 0
if self.flags.push:
self.create()
chart_begin = CHART_BEGIN.format(type=self.type,
id=self.id,
since_last=since_last)
safe_print(chart_begin, updated_dimensions, updated_variables,
'END\n')
self.flags.updated = True
self.penalty = 0
else:
self.penalty += 1
self.flags.updated = False
return bool(updated_dimensions)
def update(self, data, interval):
updated_dimensions, updated_variables = str(), str()
for dim in self.dimensions:
value = dim.get_value(data)
if value is not None:
updated_dimensions += dim.set(value)
for var in self.variables:
value = var.get_value(data)
if value is not None:
updated_variables += var.set(value)
if updated_dimensions:
since_last = interval if self.flags.updated else 0
if self.flags.push:
self.create()
chart_begin = CHART_BEGIN.format(type=self.type, id=self.id, since_last=since_last)
safe_print(chart_begin, updated_dimensions, updated_variables, 'END\n')
self.flags.updated = True
self.penalty = 0
else:
self.penalty += 1
self.flags.updated = False
return bool(updated_dimensions)
def wrapper(*args, **kwargs):
self = args[0]
ok = func(*args, **kwargs)
if ok:
safe_print(RUNTIME_CHART_CREATE.format(job_name=self.name,
update_every=self._runtime_counters.FREQ))
return ok
def add_dimension_and_push_chart(self, dimension):
"""
:param dimension: <list>
:return:
"""
dim = self.add_dimension(dimension)
self.unsuppress()
safe_print(self.create(dim))
def wrapper(*args, **kwargs):
self = args[0]
chart = RUNTIME_CHART_CREATE.format(
job_name=self.name,
update_every=self._runtime_counters.update_every,
)
safe_print(chart)
ok = func(*args, **kwargs)
return ok
def create(self):
"""
:return:
"""
chart = CHART_CREATE.format(**self.params)
dimensions = ''.join([dimension.create() for dimension in self.dimensions])
variables = ''.join([var.set(var.value) for var in self.variables if var])
self.flags.push = False
self.flags.created = True
safe_print(chart + dimensions + variables)
def update(self, interval):
"""
:return:
"""
data = self.get_data()
if not data:
self.debug('get_data() returned no data')
return False
elif not isinstance(data, dict):
self.debug('get_data() returned incorrect type data')
return False
charts_updated = False
for chart in self.charts.penalty_exceeded(penalty_max=CHART_OBSOLETE_PENALTY):
safe_print(chart.obsolete())
chart.suppress()
self.error("chart '{0}' was removed due to non updating".format(chart.name))
for chart in self.charts:
if not chart.alive:
continue
dimension_updated, variables_updated = str(), str()
for dimension in chart:
try:
value = int(data[dimension.id])
except (KeyError, TypeError):
continue
else:
dimension_updated += dimension.set(value)
for var in chart.variables:
try:
value = int(data[var.id])
except (KeyError, TypeError):
continue
else:
variables_updated += var.set(value)
if dimension_updated:
charts_updated = True
safe_print(''.join([chart.begin(since_last=interval),
dimension_updated,
variables_updated,
'END\n']))
else:
chart.penalty += 1
if not charts_updated:
self.debug('none of the charts have been updated')
return charts_updated
def run(self):
"""
Runs job in thread. Handles retries.
Exits when job failed or timed out.
:return: None
"""
job = self._runtime_counters
self.debug('started, update frequency: {freq}, '
'retries: {retries}'.format(freq=job.FREQ,
retries=job.RETRIES_MAX -
job.RETRIES))
while True:
job.START_RUN = time()
job.NEXT_RUN = job.START_RUN - (job.START_RUN %
job.FREQ) + job.FREQ + job.PENALTY
self.sleep_until_next_run()
if job.PREV_UPDATE:
job.SINCE_UPDATE = int((job.START_RUN - job.PREV_UPDATE) * 1e6)
try:
updated = self.update(interval=job.SINCE_UPDATE)
except Exception as error:
self.error('update() unhandled exception: {error}'.format(
error=error))
updated = False
job.RUNS += 1
if not updated:
if not self.manage_retries():
return
else:
job.ELAPSED = int((time() - job.START_RUN) * 1e3)
job.PREV_UPDATE = job.START_RUN
job.RETRIES, job.PENALTY = 0, 0
safe_print(
RUNTIME_CHART_UPDATE.format(job_name=self.name,
since_last=job.SINCE_UPDATE,
elapsed=job.ELAPSED))
self.debug('update => [{status}] (elapsed time: {elapsed}, '
'retries left: {retries})'.format(
status='OK' if updated else 'FAILED',
elapsed=job.ELAPSED if updated else '-',
retries=job.RETRIES_MAX - job.RETRIES))
def create(self):
for chart_name in self.order:
chart_config = self.definitions.get(chart_name)
if not chart_config:
self.debug("create() => [NOT ADDED] chart '{chart_name}' not in definitions. "
"Skipping it.".format(chart_name=chart_name))
continue
# create chart
chart_params = [chart_name] + chart_config['options']
try:
self.charts.add_chart(params=chart_params)
except ChartError as error:
self.error("create() => [NOT ADDED] (chart '{chart}': {error})".format(chart=chart_name,
error=error))
continue
# add dimensions to chart
for dimension in chart_config['lines']:
try:
self.charts[chart_name].add_dimension(dimension)
except ChartError as error:
self.error("create() => [NOT ADDED] (dimension '{dimension}': {error})".format(dimension=dimension,
error=error))
continue
# add variables to chart
if 'variables' in chart_config:
for variable in chart_config['variables']:
try:
self.charts[chart_name].add_variable(variable)
except ChartError as error:
self.error("create() => [NOT ADDED] (variable '{var}': {error})".format(var=variable,
error=error))
continue
del self.order
del self.definitions
# push charts to netdata
for chart in self.charts:
safe_print(chart.create())
# True if job has at least 1 chart else False
return bool(self.charts)
def run(self):
"""
Runs job in thread. Handles retries.
Exits when job failed or timed out.
:return: None
"""
job = self._runtime_counters
self.debug(
'started, update frequency: {freq}, retries: {retries}'.format(
freq=job.update_every,
retries=job.max_retries - job.retries), )
while True:
job.sleep_until_next()
since = 0
if job.prev_update:
since = int((job.start_real - job.prev_update) * 1e6)
try:
updated = self.update(interval=since)
except Exception as error:
self.error('update() unhandled exception: {error}'.format(
error=error))
updated = False
job.runs += 1
if not updated:
if not job.handle_retries():
return
else:
job.elapsed = int((monotonic() - job.start_mono) * 1e3)
job.prev_update = job.start_real
job.retries, job.penalty = 0, 0
safe_print(
RUNTIME_CHART_UPDATE.format(job_name=self.name,
since_last=since,
elapsed=job.elapsed))
self.debug('update => [{status}] (elapsed time: {elapsed}, '
'retries left: {retries})'.format(
status='OK' if updated else 'FAILED',
elapsed=job.elapsed if updated else '-',
retries=job.max_retries - job.retries))
def run(self):
"""
Runs job in thread. Handles retries.
Exits when job failed or timed out.
:return: None
"""
job = self._runtime_counters
self.debug('started, update frequency: {freq}, '
'retries: {retries}'.format(freq=job.FREQ, retries=job.RETRIES_MAX - job.RETRIES))
while True:
job.START_RUN = time()
job.NEXT_RUN = job.START_RUN - (job.START_RUN % job.FREQ) + job.FREQ + job.PENALTY
self.sleep_until_next_run()
if job.PREV_UPDATE:
job.SINCE_UPDATE = int((job.START_RUN - job.PREV_UPDATE) * 1e6)
try:
updated = self.update(interval=job.SINCE_UPDATE)
except Exception as error:
self.error('update() unhandled exception: {error}'.format(error=error))
updated = False
job.RUNS += 1
if not updated:
if not self.manage_retries():
return
else:
job.ELAPSED = int((time() - job.START_RUN) * 1e3)
job.PREV_UPDATE = job.START_RUN
job.RETRIES, job.PENALTY = 0, 0
safe_print(RUNTIME_CHART_UPDATE.format(job_name=self.name,
since_last=job.SINCE_UPDATE,
elapsed=job.ELAPSED))
self.debug('update => [{status}] (elapsed time: {elapsed}, '
'retries left: {retries})'.format(status='OK' if updated else 'FAILED',
elapsed=job.ELAPSED if updated else '-',
retries=job.RETRIES_MAX - job.RETRIES))
def run(self):
"""
Runs job in thread. Handles retries.
Exits when job failed or timed out.
:return: None
"""
job = self._runtime_counters
self.debug('started, update frequency: {freq}, retries: {retries}'.format(
freq=job.update_every,
retries=job.max_retries - job.retries),
)
while True:
job.sleep_until_next()
since = 0
if job.prev_update:
since = int((job.start_real - job.prev_update) * 1e6)
try:
updated = self.update(interval=since)
except Exception as error:
self.error('update() unhandled exception: {error}'.format(error=error))
updated = False
job.runs += 1
if not updated:
if not job.handle_retries():
return
else:
job.elapsed = int((monotonic() - job.start_mono) * 1e3)
job.prev_update = job.start_real
job.retries, job.penalty = 0, 0
safe_print(RUNTIME_CHART_UPDATE.format(job_name=self.name,
since_last=since,
elapsed=job.elapsed))
self.debug('update => [{status}] (elapsed time: {elapsed}, '
'retries left: {retries})'.format(status='OK' if updated else 'FAILED',
elapsed=job.elapsed if updated else '-',
retries=job.max_retries - job.retries))
def obsolete(self):
self.flags.obsoleted = True
if self.flags.created:
safe_print(CHART_OBSOLETE.format(**self.params))
def obsolete(self):
self.flags.obsoleted = True
if self.flags.created:
safe_print(CHART_OBSOLETE.format(**self.params))