def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn(_('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone as e: self.stop() done.send(e.retvalue) except Exception: LOG.exception(_('in fixed duration looping call')) done.send_exception(*sys.exc_info()) return else: done.send(True)
def _exec_usage(self): with db_session() as session: # Next check if it's time to send bandwidth usage notifications delta = datetime.timedelta(minutes=self.usage_freq) exp = timeutils.utcnow() - delta start, = session.query( Billing.last_update ).filter(Billing.name == "usage").\ first() if start and start > exp: # Not time yet LOG.info('Not time to send usage statistics yet {0}'. format(exp)) session.rollback() return # Check the connection before sending the notifications if not test_mnb_connection(): # Abort the exists notifications LOG.info("Aborting usage notifications. Could not connect") session.rollback() return # Calculate the stopping point by rounding backward to the nearest # N minutes. i.e. if N = 60, this will round us back to HH:00:00, # or if N = 15, it will round us back to HH:15:00, HH:30:00, # HH:45:00, or HH:00:00, whichever is closest. N = cfg.CONF['admin_api'].usage_freq now = timeutils.utcnow() stop = now - datetime.timedelta(minutes=now.minute % N, seconds=now.second, microseconds=now.microsecond) # Release the lock session.query(Billing).\ filter(Billing.name == "usage").\ update({"last_update": stop}, synchronize_session='fetch') session.commit() # Send the usage notifications. Pass the timestamps to save # queries. update_mnb('lbaas.bandwidth.usage', start, stop)
def _exec_stats(self): failed = 0 node_list = [] with db_session() as session: delta = datetime.timedelta(minutes=self.stats_freq) exp = timeutils.utcnow() - delta exp_time = exp.strftime('%Y-%m-%d %H:%M:%S') updated = session.query( Billing.last_update ).filter(Billing.name == "stats").\ filter(Billing.last_update > exp_time).\ first() if updated is not None: # Not time yet LOG.info('Not time to gather stats yet {0}'.format(exp_time)) session.rollback() return 0, 0 #Update the stats timestamp session.query(Billing).\ filter(Billing.name == "stats").\ update({"last_update": func.now()}, synchronize_session='fetch') # Get all the online devices to query for stats devices = session.query( Device.id, Device.name ).filter(Device.status == 'ONLINE').all() if devices is None or len(devices) == 0: LOG.error('No ONLINE devices to gather usage stats from') session.rollback() return 0, 0 total = len(devices) for device in devices: node_list.append(device.name) gearman = GearJobs() failed_list, results = gearman.get_stats(node_list) failed = len(failed_list) if failed > 0: self._send_fails(failed_list) if total > failed: # We have some success self._update_stats(results, failed_list) session.commit() else: # Everything failed. Retry these on the next timer firing session.rollback() return failed, total
def _exec_stats(self): failed = 0 node_list = [] with db_session() as session: delta = datetime.timedelta(minutes=self.stats_freq) exp = timeutils.utcnow() - delta exp_time = exp.strftime('%Y-%m-%d %H:%M:%S') updated = session.query( Billing.last_update ).filter(Billing.name == "stats").\ filter(Billing.last_update > exp_time).\ first() if updated is not None: # Not time yet LOG.info('Not time to gather stats yet {0}'.format(exp_time)) session.rollback() return 0, 0 # Update the stats timestamp session.query(Billing).\ filter(Billing.name == "stats").\ update({"last_update": func.now()}, synchronize_session='fetch') # Get all the online devices to query for stats devices = session.query( Device.id, Device.name).filter(Device.status == 'ONLINE').all() if devices is None or len(devices) == 0: LOG.error('No ONLINE devices to gather usage stats from') session.rollback() return 0, 0 total = len(devices) for device in devices: node_list.append(device.name) gearman = GearJobs() failed_list, results = gearman.get_stats(node_list) failed = len(failed_list) if failed > 0: self._send_fails(failed_list) if total > failed: # We have some success self._update_stats(results, failed_list) session.commit() else: # Everything failed. Retry these on the next timer firing session.rollback() return failed, total
def notify(context, publisher_id, event_type, priority, payload): """Sends a notification using the specified driver :param publisher_id: the source worker_type.host of the message :param event_type: the literal type of event (ex. Instance Creation) :param priority: patterned after the enumeration of Python logging levels in the set (DEBUG, WARN, INFO, ERROR, CRITICAL) :param payload: A python dictionary of attributes Outgoing message format includes the above parameters, and appends the following: message_id a UUID representing the id for this notification timestamp the GMT timestamp the notification was sent at The composite message will be constructed as a dictionary of the above attributes, which will then be sent via the transport mechanism defined by the driver. Message example:: {'message_id': str(uuid.uuid4()), 'publisher_id': 'compute.host1', 'timestamp': timeutils.utcnow(), 'priority': 'WARN', 'event_type': 'compute.create_instance', 'payload': {'instance_id': 12, ... }} """ if priority not in log_levels: raise BadPriorityException( _('%s not in valid priorities') % priority) # Ensure everything is JSON serializable. payload = jsonutils.to_primitive(payload, convert_instances=True) msg = dict(message_id=str(uuid.uuid4()), publisher_id=publisher_id, event_type=event_type, priority=priority, payload=payload, timestamp=str(timeutils.utcnow())) for driver in _get_drivers(): try: driver.notify(context, msg) except Exception as e: LOG.exception(_("Problem '%(e)s' attempting to " "send to notification system. " "Payload=%(payload)s") % dict(e=e, payload=payload))
def notify(context, publisher_id, event_type, priority, payload): """Sends a notification using the specified driver :param publisher_id: the source worker_type.host of the message :param event_type: the literal type of event (ex. Instance Creation) :param priority: patterned after the enumeration of Python logging levels in the set (DEBUG, WARN, INFO, ERROR, CRITICAL) :param payload: A python dictionary of attributes Outgoing message format includes the above parameters, and appends the following: message_id a UUID representing the id for this notification timestamp the GMT timestamp the notification was sent at The composite message will be constructed as a dictionary of the above attributes, which will then be sent via the transport mechanism defined by the driver. Message example:: {'message_id': str(uuid.uuid4()), 'publisher_id': 'compute.host1', 'timestamp': timeutils.utcnow(), 'priority': 'WARN', 'event_type': 'compute.create_instance', 'payload': {'instance_id': 12, ... }} """ if priority not in log_levels: raise BadPriorityException(_('%s not in valid priorities') % priority) # Ensure everything is JSON serializable. payload = jsonutils.to_primitive(payload, convert_instances=True) msg = dict(message_id=str(uuid.uuid4()), publisher_id=publisher_id, event_type=event_type, priority=priority, payload=payload, timestamp=str(timeutils.utcnow())) for driver in _get_drivers(): try: driver.notify(context, msg) except Exception as e: LOG.exception( _("Problem '%(e)s' attempting to " "send to notification system. " "Payload=%(payload)s") % dict(e=e, payload=payload))
def _send_test(event_type, lbid, tenant_id): # Build the payload now = str(timeutils.utcnow()) LOG.error("Sending {0} test notifications".format(lbid)) if not test_mnb_connection(): # Abort the test notifications LOG.info("Aborting test Notifications. Could not connect") return #Note lbid is the number of notifications to send lbid += 1 for x in xrange(1, lbid): payload = _build_payload(now, now, "Test LB", str(x), str(tenant_id), 'active') _notify('lbaas', 'lbaas.instance.test', payload)
def _send_test(event_type, lbid, tenant_id): # Build the payload now = str(timeutils.utcnow()) LOG.error("Sending {0} test notifications".format(lbid)) if not test_mnb_connection(): # Abort the test notifications LOG.info("Aborting test Notifications. Could not connect") return # Note lbid is the number of notifications to send lbid += 1 for x in xrange(1, lbid): payload = _build_payload(now, now, "Test LB", str(x), str(tenant_id), 'active') _notify('lbaas', 'lbaas.instance.test', payload)
def _send_exists(event_type): LOG.info("Sending MnB {0} notifications to MnB".format(event_type)) count = 0 with db_session() as session: lbs = session.query( LoadBalancer.id, LoadBalancer.tenantid, LoadBalancer.name, LoadBalancer.status, LoadBalancer.created, LoadBalancer.updated ).filter(LoadBalancer.status != 'DELETED').all() if not lbs: session.rollback() LOG.error("No existing Load Balancers found") return # Figure out our audit period beging/ending seconds = (CONF['admin_api']['exists_freq'] * 60) interval = datetime.timedelta(seconds=seconds) audit_period_ending = timeutils.utcnow() audit_period_beginning = audit_period_ending - interval audit_period_beginning = str(audit_period_beginning) audit_period_ending = str(audit_period_ending) for lb in lbs: LOG.info( "Sending MnB {0} notification to MnB for " "loadbalancer {1} tenant_id {2}".format( event_type, lb.id, lb.tenantid) ) # Build the payload payload = _build_payload(audit_period_beginning, audit_period_ending, lb.name, lb.id, lb.tenantid, lb.status) _notify('lbaas', event_type, payload) count += 1 session.commit() LOG.info("Sent {0} MnB {1} notifications to MnB".format(count, event_type))
def _send_exists(event_type): LOG.info("Sending MnB {0} notifications to MnB".format(event_type)) count = 0 with db_session() as session: lbs = session.query(LoadBalancer.id, LoadBalancer.tenantid, LoadBalancer.name, LoadBalancer.status, LoadBalancer.created, LoadBalancer.updated).filter( LoadBalancer.status != 'DELETED').all() if not lbs: session.rollback() LOG.error("No existing Load Balancers found") return # Figure out our audit period beging/ending seconds = (CONF['admin_api']['exists_freq'] * 60) interval = datetime.timedelta(seconds=seconds) audit_period_ending = timeutils.utcnow() audit_period_beginning = audit_period_ending - interval audit_period_beginning = str(audit_period_beginning) audit_period_ending = str(audit_period_ending) for lb in lbs: LOG.info("Sending MnB {0} notification to MnB for " "loadbalancer {1} tenant_id {2}".format( event_type, lb.id, lb.tenantid)) # Build the payload payload = _build_payload(audit_period_beginning, audit_period_ending, lb.name, lb.id, lb.tenantid, lb.status) _notify('lbaas', event_type, payload) count += 1 session.commit() LOG.info("Sent {0} MnB {1} notifications to MnB".format(count, event_type))
def _exec_exists(self): with db_session() as session: # Check if it's time to send exists notifications delta = datetime.timedelta(minutes=self.exists_freq) exp = timeutils.utcnow() - delta exp_time = exp.strftime('%Y-%m-%d %H:%M:%S') updated = session.query( Billing.last_update ).filter(Billing.name == "exists").\ filter(Billing.last_update > exp_time).\ first() if updated is not None: # Not time yet LOG.info('Not time to send exists notifications yet {0}'. format(exp_time)) session.rollback() return # Check the connection before sending the notifications if not test_mnb_connection(): # Abort the exists notifications LOG.info("Aborting exists notifications. Could not connect") session.rollback() return #Update the exists timestamp now session.query(Billing).\ filter(Billing.name == "exists").\ update({"last_update": func.now()}, synchronize_session='fetch') session.commit() # Send the notifications update_mnb('lbaas.instance.exists', None, None)
def _send_usage(event_type, start, stop): LOG.info("Sending MnB {0} notifications to MnB".format(event_type)) N = CONF['admin_api']['usage_freq'] with db_session() as session: # Start by making sure we have stats in the Stats table and # track the oldest value in case we need it below. oldest, = session.query(Stats.period_end).\ order_by(Stats.id.asc()).first() if oldest is None: # No Stats at all LOG.info("No usage statistics to send.") session.rollback() return if start is None: # The value in the DB must be '0000-00-00 00:00:00 so # as a starting point, we can find the oldest stat in # the Stats table and start from there. No sense iterating # from 0000-00-00 to now looking for stats to send. Also # round it back to the previous update period start = _rounded_down_min(oldest, N) LOG.info("Starting usage notifications from first saved {0}". format(start)) # Now that we know where to start, make sure we have stats to # send for the time period. Use stats that end in this period. # It's ok if the stats started in a previous period. Some skew # is allowed. total = session.query(Stats).\ filter(Stats.period_end >= start).\ filter(Stats.period_end < stop).\ count() if total == 0: LOG.info("No usage statistics to send between {0} and {1}" .format(start, stop)) session.rollback() return LOG.info("Found {0} total usage statistics to send between {1} and {2}" .format(total, start, stop)) # Get info on all of our loadbalancers for the payloads. loadbalancers = _get_lbs() # Get ready to loop through however N minute periods we # have to send. We do it this way rather than one lump sum # because finer grain data is probably needed on the MnB side. end = start + datetime.timedelta(minutes=N) count = 0 while end <= stop: # Loop through all N periods up to the current period # sending usage notifications to MnB stats = session.query( Stats.lbid, func.sum(Stats.bytes_out) ).group_by(Stats.lbid).\ filter(Stats.period_end >= start).\ filter(Stats.period_end < end).\ all() # Prep for the next loop here in case of continue prev_start = start prev_end = end start = end end = start + datetime.timedelta(minutes=N) if not stats: LOG.info("No usage statistics to send for period {0} to {1}". format(prev_start, prev_end)) continue else: LOG.info("Sending usage statistics for {0} to {1}". format(prev_start, prev_end)) audit_period_beginning = str(prev_start) audit_period_ending = str(prev_end) for lb in stats: lbid, byte_count = lb byte_count = int(byte_count) if lbid not in loadbalancers: LOG.error("Loadbalancer {0} not found in DB " "not sending usage statistics".format(lbid)) continue # Build the payload payload = _build_payload(audit_period_beginning, audit_period_ending, loadbalancers[lbid]["name"], lbid, loadbalancers[lbid]["tenant_id"], loadbalancers[lbid]["status"]) payload["metrics"] = _build_metrics(byte_count) LOG.info( "Sending MnB {0} notification to MnB for " "loadbalancer {1} tenant_id {2} from " "{3} to {4}: PAYLOAD = {5}". format(event_type, lbid, loadbalancers[lbid]["tenant_id"], prev_start, prev_end, payload) ) _notify('lbaas', event_type, payload) count += 1 # Purge old stats if CONF['admin_api']['stats_purge_enable']: hours = CONF['admin_api']['stats_purge_days'] * 24 delta = datetime.timedelta(hours=hours) exp = timeutils.utcnow() - delta exp_time = exp.strftime('%Y-%m-%d %H:%M:%S') purged = session.query(Stats).\ filter(Stats.period_end < exp_time).\ delete() LOG.info("Purged {0} usage statistics from before {1}". format(purged, exp_time)) session.commit() LOG.info("Sent {0} MnB {1} notifications to MnB".format(count, event_type))
def _send_usage(event_type, start, stop): LOG.info("Sending MnB {0} notifications to MnB".format(event_type)) N = CONF['admin_api']['usage_freq'] with db_session() as session: # Start by making sure we have stats in the Stats table and # track the oldest value in case we need it below. oldest, = session.query(Stats.period_end).\ order_by(Stats.id.asc()).first() if oldest is None: # No Stats at all LOG.info("No usage statistics to send.") session.rollback() return if start is None: # The value in the DB must be '0000-00-00 00:00:00 so # as a starting point, we can find the oldest stat in # the Stats table and start from there. No sense iterating # from 0000-00-00 to now looking for stats to send. Also # round it back to the previous update period start = _rounded_down_min(oldest, N) LOG.info( "Starting usage notifications from first saved {0}".format( start)) # Now that we know where to start, make sure we have stats to # send for the time period. Use stats that end in this period. # It's ok if the stats started in a previous period. Some skew # is allowed. total = session.query(Stats).\ filter(Stats.period_end >= start).\ filter(Stats.period_end < stop).\ count() if total == 0: LOG.info("No usage statistics to send between {0} and {1}".format( start, stop)) session.rollback() return LOG.info( "Found {0} total usage statistics to send between {1} and {2}". format(total, start, stop)) # Get info on all of our loadbalancers for the payloads. loadbalancers = _get_lbs() # Get ready to loop through however N minute periods we # have to send. We do it this way rather than one lump sum # because finer grain data is probably needed on the MnB side. end = start + datetime.timedelta(minutes=N) count = 0 while end <= stop: # Loop through all N periods up to the current period # sending usage notifications to MnB stats = session.query( Stats.lbid, func.sum(Stats.bytes_out) ).group_by(Stats.lbid).\ filter(Stats.period_end >= start).\ filter(Stats.period_end < end).\ all() # Prep for the next loop here in case of continue prev_start = start prev_end = end start = end end = start + datetime.timedelta(minutes=N) if not stats: LOG.info( "No usage statistics to send for period {0} to {1}".format( prev_start, prev_end)) continue else: LOG.info("Sending usage statistics for {0} to {1}".format( prev_start, prev_end)) audit_period_beginning = str(prev_start) audit_period_ending = str(prev_end) for lb in stats: lbid, byte_count = lb byte_count = int(byte_count) if lbid not in loadbalancers: LOG.error("Loadbalancer {0} not found in DB " "not sending usage statistics".format(lbid)) continue # Build the payload payload = _build_payload(audit_period_beginning, audit_period_ending, loadbalancers[lbid]["name"], lbid, loadbalancers[lbid]["tenant_id"], loadbalancers[lbid]["status"]) payload["metrics"] = _build_metrics(byte_count) LOG.info("Sending MnB {0} notification to MnB for " "loadbalancer {1} tenant_id {2} from " "{3} to {4}: PAYLOAD = {5}".format( event_type, lbid, loadbalancers[lbid]["tenant_id"], prev_start, prev_end, payload)) _notify('lbaas', event_type, payload) count += 1 # Purge old stats if CONF['admin_api']['stats_purge_enable']: hours = CONF['admin_api']['stats_purge_days'] * 24 delta = datetime.timedelta(hours=hours) exp = timeutils.utcnow() - delta exp_time = exp.strftime('%Y-%m-%d %H:%M:%S') purged = session.query(Stats).\ filter(Stats.period_end < exp_time).\ delete() LOG.info("Purged {0} usage statistics from before {1}".format( purged, exp_time)) session.commit() LOG.info("Sent {0} MnB {1} notifications to MnB".format(count, event_type))