def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running:
break
delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(_('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
except Exception:
LOG.exception(_('in fixed duration looping call'))
done.send_exception(*sys.exc_info())
return
else:
done.send(True)
def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running:
break
delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(_('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
except Exception:
LOG.exception(_('in fixed duration looping call'))
done.send_exception(*sys.exc_info())
return
else:
done.send(True)
def _exec_usage(self):
with db_session() as session:
# Next check if it's time to send bandwidth usage notifications
delta = datetime.timedelta(minutes=self.usage_freq)
exp = timeutils.utcnow() - delta
start, = session.query(
Billing.last_update
).filter(Billing.name == "usage").\
first()
if start and start > exp:
# Not time yet
LOG.info('Not time to send usage statistics yet {0}'.
format(exp))
session.rollback()
return
# Check the connection before sending the notifications
if not test_mnb_connection():
# Abort the exists notifications
LOG.info("Aborting usage notifications. Could not connect")
session.rollback()
return
# Calculate the stopping point by rounding backward to the nearest
# N minutes. i.e. if N = 60, this will round us back to HH:00:00,
# or if N = 15, it will round us back to HH:15:00, HH:30:00,
# HH:45:00, or HH:00:00, whichever is closest.
N = cfg.CONF['admin_api'].usage_freq
now = timeutils.utcnow()
stop = now - datetime.timedelta(minutes=now.minute % N,
seconds=now.second,
microseconds=now.microsecond)
# Release the lock
session.query(Billing).\
filter(Billing.name == "usage").\
update({"last_update": stop},
synchronize_session='fetch')
session.commit()
# Send the usage notifications. Pass the timestamps to save
# queries.
update_mnb('lbaas.bandwidth.usage', start, stop)
def _exec_stats(self):
failed = 0
node_list = []
with db_session() as session:
delta = datetime.timedelta(minutes=self.stats_freq)
exp = timeutils.utcnow() - delta
exp_time = exp.strftime('%Y-%m-%d %H:%M:%S')
updated = session.query(
Billing.last_update
).filter(Billing.name == "stats").\
filter(Billing.last_update > exp_time).\
first()
if updated is not None:
# Not time yet
LOG.info('Not time to gather stats yet {0}'.format(exp_time))
session.rollback()
return 0, 0
#Update the stats timestamp
session.query(Billing).\
filter(Billing.name == "stats").\
update({"last_update": func.now()},
synchronize_session='fetch')
# Get all the online devices to query for stats
devices = session.query(
Device.id, Device.name
).filter(Device.status == 'ONLINE').all()
if devices is None or len(devices) == 0:
LOG.error('No ONLINE devices to gather usage stats from')
session.rollback()
return 0, 0
total = len(devices)
for device in devices:
node_list.append(device.name)
gearman = GearJobs()
failed_list, results = gearman.get_stats(node_list)
failed = len(failed_list)
if failed > 0:
self._send_fails(failed_list)
if total > failed:
# We have some success
self._update_stats(results, failed_list)
session.commit()
else:
# Everything failed. Retry these on the next timer firing
session.rollback()
return failed, total
def _exec_stats(self):
failed = 0
node_list = []
with db_session() as session:
delta = datetime.timedelta(minutes=self.stats_freq)
exp = timeutils.utcnow() - delta
exp_time = exp.strftime('%Y-%m-%d %H:%M:%S')
updated = session.query(
Billing.last_update
).filter(Billing.name == "stats").\
filter(Billing.last_update > exp_time).\
first()
if updated is not None:
# Not time yet
LOG.info('Not time to gather stats yet {0}'.format(exp_time))
session.rollback()
return 0, 0
# Update the stats timestamp
session.query(Billing).\
filter(Billing.name == "stats").\
update({"last_update": func.now()},
synchronize_session='fetch')
# Get all the online devices to query for stats
devices = session.query(
Device.id,
Device.name).filter(Device.status == 'ONLINE').all()
if devices is None or len(devices) == 0:
LOG.error('No ONLINE devices to gather usage stats from')
session.rollback()
return 0, 0
total = len(devices)
for device in devices:
node_list.append(device.name)
gearman = GearJobs()
failed_list, results = gearman.get_stats(node_list)
failed = len(failed_list)
if failed > 0:
self._send_fails(failed_list)
if total > failed:
# We have some success
self._update_stats(results, failed_list)
session.commit()
else:
# Everything failed. Retry these on the next timer firing
session.rollback()
return failed, total
def notify(context, publisher_id, event_type, priority, payload):
"""Sends a notification using the specified driver
:param publisher_id: the source worker_type.host of the message
:param event_type: the literal type of event (ex. Instance Creation)
:param priority: patterned after the enumeration of Python logging
levels in the set (DEBUG, WARN, INFO, ERROR, CRITICAL)
:param payload: A python dictionary of attributes
Outgoing message format includes the above parameters, and appends the
following:
message_id
a UUID representing the id for this notification
timestamp
the GMT timestamp the notification was sent at
The composite message will be constructed as a dictionary of the above
attributes, which will then be sent via the transport mechanism defined
by the driver.
Message example::
{'message_id': str(uuid.uuid4()),
'publisher_id': 'compute.host1',
'timestamp': timeutils.utcnow(),
'priority': 'WARN',
'event_type': 'compute.create_instance',
'payload': {'instance_id': 12, ... }}
"""
if priority not in log_levels:
raise BadPriorityException(
_('%s not in valid priorities') % priority)
# Ensure everything is JSON serializable.
payload = jsonutils.to_primitive(payload, convert_instances=True)
msg = dict(message_id=str(uuid.uuid4()),
publisher_id=publisher_id,
event_type=event_type,
priority=priority,
payload=payload,
timestamp=str(timeutils.utcnow()))
for driver in _get_drivers():
try:
driver.notify(context, msg)
except Exception as e:
LOG.exception(_("Problem '%(e)s' attempting to "
"send to notification system. "
"Payload=%(payload)s")
% dict(e=e, payload=payload))
def notify(context, publisher_id, event_type, priority, payload):
"""Sends a notification using the specified driver
:param publisher_id: the source worker_type.host of the message
:param event_type: the literal type of event (ex. Instance Creation)
:param priority: patterned after the enumeration of Python logging
levels in the set (DEBUG, WARN, INFO, ERROR, CRITICAL)
:param payload: A python dictionary of attributes
Outgoing message format includes the above parameters, and appends the
following:
message_id
a UUID representing the id for this notification
timestamp
the GMT timestamp the notification was sent at
The composite message will be constructed as a dictionary of the above
attributes, which will then be sent via the transport mechanism defined
by the driver.
Message example::
{'message_id': str(uuid.uuid4()),
'publisher_id': 'compute.host1',
'timestamp': timeutils.utcnow(),
'priority': 'WARN',
'event_type': 'compute.create_instance',
'payload': {'instance_id': 12, ... }}
"""
if priority not in log_levels:
raise BadPriorityException(_('%s not in valid priorities') % priority)
# Ensure everything is JSON serializable.
payload = jsonutils.to_primitive(payload, convert_instances=True)
msg = dict(message_id=str(uuid.uuid4()),
publisher_id=publisher_id,
event_type=event_type,
priority=priority,
payload=payload,
timestamp=str(timeutils.utcnow()))
for driver in _get_drivers():
try:
driver.notify(context, msg)
except Exception as e:
LOG.exception(
_("Problem '%(e)s' attempting to "
"send to notification system. "
"Payload=%(payload)s") % dict(e=e, payload=payload))
def _send_test(event_type, lbid, tenant_id):
# Build the payload
now = str(timeutils.utcnow())
LOG.error("Sending {0} test notifications".format(lbid))
if not test_mnb_connection():
# Abort the test notifications
LOG.info("Aborting test Notifications. Could not connect")
return
#Note lbid is the number of notifications to send
lbid += 1
for x in xrange(1, lbid):
payload = _build_payload(now, now, "Test LB", str(x),
str(tenant_id), 'active')
_notify('lbaas', 'lbaas.instance.test', payload)
def _send_test(event_type, lbid, tenant_id):
# Build the payload
now = str(timeutils.utcnow())
LOG.error("Sending {0} test notifications".format(lbid))
if not test_mnb_connection():
# Abort the test notifications
LOG.info("Aborting test Notifications. Could not connect")
return
# Note lbid is the number of notifications to send
lbid += 1
for x in xrange(1, lbid):
payload = _build_payload(now, now, "Test LB", str(x), str(tenant_id),
'active')
_notify('lbaas', 'lbaas.instance.test', payload)
def _send_exists(event_type):
LOG.info("Sending MnB {0} notifications to MnB".format(event_type))
count = 0
with db_session() as session:
lbs = session.query(
LoadBalancer.id,
LoadBalancer.tenantid,
LoadBalancer.name,
LoadBalancer.status,
LoadBalancer.created,
LoadBalancer.updated
).filter(LoadBalancer.status != 'DELETED').all()
if not lbs:
session.rollback()
LOG.error("No existing Load Balancers found")
return
# Figure out our audit period beging/ending
seconds = (CONF['admin_api']['exists_freq'] * 60)
interval = datetime.timedelta(seconds=seconds)
audit_period_ending = timeutils.utcnow()
audit_period_beginning = audit_period_ending - interval
audit_period_beginning = str(audit_period_beginning)
audit_period_ending = str(audit_period_ending)
for lb in lbs:
LOG.info(
"Sending MnB {0} notification to MnB for "
"loadbalancer {1} tenant_id {2}".format(
event_type, lb.id, lb.tenantid)
)
# Build the payload
payload = _build_payload(audit_period_beginning,
audit_period_ending,
lb.name, lb.id, lb.tenantid, lb.status)
_notify('lbaas', event_type, payload)
count += 1
session.commit()
LOG.info("Sent {0} MnB {1} notifications to MnB".format(count, event_type))
def _send_exists(event_type):
LOG.info("Sending MnB {0} notifications to MnB".format(event_type))
count = 0
with db_session() as session:
lbs = session.query(LoadBalancer.id, LoadBalancer.tenantid,
LoadBalancer.name, LoadBalancer.status,
LoadBalancer.created, LoadBalancer.updated).filter(
LoadBalancer.status != 'DELETED').all()
if not lbs:
session.rollback()
LOG.error("No existing Load Balancers found")
return
# Figure out our audit period beging/ending
seconds = (CONF['admin_api']['exists_freq'] * 60)
interval = datetime.timedelta(seconds=seconds)
audit_period_ending = timeutils.utcnow()
audit_period_beginning = audit_period_ending - interval
audit_period_beginning = str(audit_period_beginning)
audit_period_ending = str(audit_period_ending)
for lb in lbs:
LOG.info("Sending MnB {0} notification to MnB for "
"loadbalancer {1} tenant_id {2}".format(
event_type, lb.id, lb.tenantid))
# Build the payload
payload = _build_payload(audit_period_beginning,
audit_period_ending, lb.name, lb.id,
lb.tenantid, lb.status)
_notify('lbaas', event_type, payload)
count += 1
session.commit()
LOG.info("Sent {0} MnB {1} notifications to MnB".format(count, event_type))
def _exec_exists(self):
with db_session() as session:
# Check if it's time to send exists notifications
delta = datetime.timedelta(minutes=self.exists_freq)
exp = timeutils.utcnow() - delta
exp_time = exp.strftime('%Y-%m-%d %H:%M:%S')
updated = session.query(
Billing.last_update
).filter(Billing.name == "exists").\
filter(Billing.last_update > exp_time).\
first()
if updated is not None:
# Not time yet
LOG.info('Not time to send exists notifications yet {0}'.
format(exp_time))
session.rollback()
return
# Check the connection before sending the notifications
if not test_mnb_connection():
# Abort the exists notifications
LOG.info("Aborting exists notifications. Could not connect")
session.rollback()
return
#Update the exists timestamp now
session.query(Billing).\
filter(Billing.name == "exists").\
update({"last_update": func.now()},
synchronize_session='fetch')
session.commit()
# Send the notifications
update_mnb('lbaas.instance.exists', None, None)
def _send_usage(event_type, start, stop):
LOG.info("Sending MnB {0} notifications to MnB".format(event_type))
N = CONF['admin_api']['usage_freq']
with db_session() as session:
# Start by making sure we have stats in the Stats table and
# track the oldest value in case we need it below.
oldest, = session.query(Stats.period_end).\
order_by(Stats.id.asc()).first()
if oldest is None:
# No Stats at all
LOG.info("No usage statistics to send.")
session.rollback()
return
if start is None:
# The value in the DB must be '0000-00-00 00:00:00 so
# as a starting point, we can find the oldest stat in
# the Stats table and start from there. No sense iterating
# from 0000-00-00 to now looking for stats to send. Also
# round it back to the previous update period
start = _rounded_down_min(oldest, N)
LOG.info("Starting usage notifications from first saved {0}".
format(start))
# Now that we know where to start, make sure we have stats to
# send for the time period. Use stats that end in this period.
# It's ok if the stats started in a previous period. Some skew
# is allowed.
total = session.query(Stats).\
filter(Stats.period_end >= start).\
filter(Stats.period_end < stop).\
count()
if total == 0:
LOG.info("No usage statistics to send between {0} and {1}"
.format(start, stop))
session.rollback()
return
LOG.info("Found {0} total usage statistics to send between {1} and {2}"
.format(total, start, stop))
# Get info on all of our loadbalancers for the payloads.
loadbalancers = _get_lbs()
# Get ready to loop through however N minute periods we
# have to send. We do it this way rather than one lump sum
# because finer grain data is probably needed on the MnB side.
end = start + datetime.timedelta(minutes=N)
count = 0
while end <= stop:
# Loop through all N periods up to the current period
# sending usage notifications to MnB
stats = session.query(
Stats.lbid,
func.sum(Stats.bytes_out)
).group_by(Stats.lbid).\
filter(Stats.period_end >= start).\
filter(Stats.period_end < end).\
all()
# Prep for the next loop here in case of continue
prev_start = start
prev_end = end
start = end
end = start + datetime.timedelta(minutes=N)
if not stats:
LOG.info("No usage statistics to send for period {0} to {1}".
format(prev_start, prev_end))
continue
else:
LOG.info("Sending usage statistics for {0} to {1}".
format(prev_start, prev_end))
audit_period_beginning = str(prev_start)
audit_period_ending = str(prev_end)
for lb in stats:
lbid, byte_count = lb
byte_count = int(byte_count)
if lbid not in loadbalancers:
LOG.error("Loadbalancer {0} not found in DB "
"not sending usage statistics".format(lbid))
continue
# Build the payload
payload = _build_payload(audit_period_beginning,
audit_period_ending,
loadbalancers[lbid]["name"],
lbid,
loadbalancers[lbid]["tenant_id"],
loadbalancers[lbid]["status"])
payload["metrics"] = _build_metrics(byte_count)
LOG.info(
"Sending MnB {0} notification to MnB for "
"loadbalancer {1} tenant_id {2} from "
"{3} to {4}: PAYLOAD = {5}".
format(event_type,
lbid,
loadbalancers[lbid]["tenant_id"],
prev_start,
prev_end,
payload)
)
_notify('lbaas', event_type, payload)
count += 1
# Purge old stats
if CONF['admin_api']['stats_purge_enable']:
hours = CONF['admin_api']['stats_purge_days'] * 24
delta = datetime.timedelta(hours=hours)
exp = timeutils.utcnow() - delta
exp_time = exp.strftime('%Y-%m-%d %H:%M:%S')
purged = session.query(Stats).\
filter(Stats.period_end < exp_time).\
delete()
LOG.info("Purged {0} usage statistics from before {1}".
format(purged, exp_time))
session.commit()
LOG.info("Sent {0} MnB {1} notifications to MnB".format(count, event_type))
def _send_usage(event_type, start, stop):
LOG.info("Sending MnB {0} notifications to MnB".format(event_type))
N = CONF['admin_api']['usage_freq']
with db_session() as session:
# Start by making sure we have stats in the Stats table and
# track the oldest value in case we need it below.
oldest, = session.query(Stats.period_end).\
order_by(Stats.id.asc()).first()
if oldest is None:
# No Stats at all
LOG.info("No usage statistics to send.")
session.rollback()
return
if start is None:
# The value in the DB must be '0000-00-00 00:00:00 so
# as a starting point, we can find the oldest stat in
# the Stats table and start from there. No sense iterating
# from 0000-00-00 to now looking for stats to send. Also
# round it back to the previous update period
start = _rounded_down_min(oldest, N)
LOG.info(
"Starting usage notifications from first saved {0}".format(
start))
# Now that we know where to start, make sure we have stats to
# send for the time period. Use stats that end in this period.
# It's ok if the stats started in a previous period. Some skew
# is allowed.
total = session.query(Stats).\
filter(Stats.period_end >= start).\
filter(Stats.period_end < stop).\
count()
if total == 0:
LOG.info("No usage statistics to send between {0} and {1}".format(
start, stop))
session.rollback()
return
LOG.info(
"Found {0} total usage statistics to send between {1} and {2}".
format(total, start, stop))
# Get info on all of our loadbalancers for the payloads.
loadbalancers = _get_lbs()
# Get ready to loop through however N minute periods we
# have to send. We do it this way rather than one lump sum
# because finer grain data is probably needed on the MnB side.
end = start + datetime.timedelta(minutes=N)
count = 0
while end <= stop:
# Loop through all N periods up to the current period
# sending usage notifications to MnB
stats = session.query(
Stats.lbid,
func.sum(Stats.bytes_out)
).group_by(Stats.lbid).\
filter(Stats.period_end >= start).\
filter(Stats.period_end < end).\
all()
# Prep for the next loop here in case of continue
prev_start = start
prev_end = end
start = end
end = start + datetime.timedelta(minutes=N)
if not stats:
LOG.info(
"No usage statistics to send for period {0} to {1}".format(
prev_start, prev_end))
continue
else:
LOG.info("Sending usage statistics for {0} to {1}".format(
prev_start, prev_end))
audit_period_beginning = str(prev_start)
audit_period_ending = str(prev_end)
for lb in stats:
lbid, byte_count = lb
byte_count = int(byte_count)
if lbid not in loadbalancers:
LOG.error("Loadbalancer {0} not found in DB "
"not sending usage statistics".format(lbid))
continue
# Build the payload
payload = _build_payload(audit_period_beginning,
audit_period_ending,
loadbalancers[lbid]["name"], lbid,
loadbalancers[lbid]["tenant_id"],
loadbalancers[lbid]["status"])
payload["metrics"] = _build_metrics(byte_count)
LOG.info("Sending MnB {0} notification to MnB for "
"loadbalancer {1} tenant_id {2} from "
"{3} to {4}: PAYLOAD = {5}".format(
event_type, lbid,
loadbalancers[lbid]["tenant_id"], prev_start,
prev_end, payload))
_notify('lbaas', event_type, payload)
count += 1
# Purge old stats
if CONF['admin_api']['stats_purge_enable']:
hours = CONF['admin_api']['stats_purge_days'] * 24
delta = datetime.timedelta(hours=hours)
exp = timeutils.utcnow() - delta
exp_time = exp.strftime('%Y-%m-%d %H:%M:%S')
purged = session.query(Stats).\
filter(Stats.period_end < exp_time).\
delete()
LOG.info("Purged {0} usage statistics from before {1}".format(
purged, exp_time))
session.commit()
LOG.info("Sent {0} MnB {1} notifications to MnB".format(count, event_type))
