def notifyZopeApplicationOpenedSubscribers(event):
"""
Re-fires the IDatabaseOpenedWithRoot notification to subscribers with an
open handle to the application defined in the database.
"""
db = event.database
conn = db.open()
try:
app = conn.root()['Application']
transact(notify)(ZopeApplicationOpenedEvent(app))
finally:
conn.close()
def notifyZopeApplicationOpenedSubscribers(event):
"""
Re-fires the IDatabaseOpenedWithRoot notification to subscribers with an
open handle to the application defined in the database.
"""
db = event.database
conn = db.open()
try:
app = conn.root()['Application']
transact(notify)(ZopeApplicationOpenedEvent(app))
finally:
conn.close()
def scan_catalog(catalog, fix):
"""Scan through a catalog looking for broken references"""
try:
brains = catalog()
sizeOfCatalog = len(brains)
scannedCount = 0
progressBarChunkSize = 1
objectsWithIssues = []
if not fix:
print "Scanning %s [%s items]:" % (catalog.id, sizeOfCatalog)
else:
print "Fixing %s [%s items]:" % (catalog.id, sizeOfCatalog)
if (sizeOfCatalog > 50):
progressBarChunkSize = (sizeOfCatalog//50) + 1
for brain in brains:
if (scannedCount % progressBarChunkSize) == 0:
sys.stdout.write('\r')
chunkNumber = scannedCount // progressBarChunkSize
sys.stdout.write("[%-50s] %d%%" % ('='*chunkNumber, 2*chunkNumber))
sys.stdout.flush()
scannedCount += 1
try:
testReference = brain.getObject()
except Exception:
objectsWithIssues.append(brain.getPath())
if fix:
transact(catalog.uncatalog_object)(brain.getPath())
# Finish off the execution progress bar since we're finished
sys.stdout.write('\r')
sys.stdout.write("[%-50s] %d%%" % ('='*50, 100))
sys.stdout.flush()
print ""
if len(objectsWithIssues) == 0:
summaryMsg.append("[Processed %s items in the %s catalog and found zero errors]"
% (scannedCount, catalog.id))
else:
summaryMsg.append("[Processed %s items in the %s catalog and found %s issues]"
% (scannedCount, catalog.id, len(objectsWithIssues)))
for item in objectsWithIssues:
summaryMsg.append(" %s" % (item))
except AttributeError:
print "{} not found. skipping....".format(catalog)
except Exception:
raise
def applyDataMap(
self,
device,
datamap,
relname="",
compname="",
modname="",
parentId="",
commit=True,
):
"""Apply a datamap passed as a list of dicts through XML-RPC,
A RelatinshipMap, or an ObjectMap
Apply datamap to device. Return True if datamap changed device.
The default value for commit is True for backwards-compatibility
reasons. If you're a new caller to ApplyDataMap._applyData you should
probably set commit to False and handle your own transactions.
@type device: Device
@param device: Device to be updated by a RelationshipMap,
or parent device of an ObjectMap.
@type datamap: RelationshipMap, ObjectMap
@param datamap: map used to update the device, and its components
@return: True if updated, False if not
"""
log.debug('requested applyDataMap for device=%s', device)
notify(DatamapAddEvent(self._dmd, datamap, device))
if not device or _locked_from_updates(device):
log.warn('device is locked from updates: device=%s', device)
return False
datamap = _validate_datamap(device,
datamap,
relname=relname,
compname=compname,
modname=modname,
parentId=parentId)
# Preprocess datamap, setting directive and diff
if isinstance(datamap, RelationshipMap):
datamap = _process_relationshipmap(datamap, device)
if not datamap:
return False
adm_method = self._apply_relationshipmap
else:
adm_method = self._apply_incrementalmap
# apply the changes
if commit:
result = transact(adm_method)(datamap, device)
else:
result = adm_method(datamap, device)
# report the changes made
result = self._report_changes(datamap, device)
log.debug('applyDataMap result=%s', result)
return result
def remove(self, dmd, leaveObjects=False):
# since this ZP added addition eventClasses, and zencatalogservice,
# if is running, indexed them, the event catalog needs to be
# cleaned up at removal
super(ZenPack, self).remove(dmd, leaveObjects=leaveObjects)
from ZODB.transact import transact
brains = dmd.Events.eventClassSearch()
for brain in brains:
try:
test_reference = brain.getObject()
test_reference._p_deactivate()
except Exception:
object_path_string = brain.getPath()
try:
transact(dmd.Events.eventClassSearch.uncatalog_object)(
object_path_string)
except Exception as e:
pass
def cleanZodb(dmd):
global_catalog = dmd.zport.global_catalog
uncat = transact(global_catalog.uncatalog_object)
for brain in global_catalog():
try:
obj = brain.getObject()
except Exception:
log.warn("Found unresolvable path, deleting: %s", brain.getPath())
uncat(brain.getPath())
log.info("Finished scanning catalog")
def cleanZodb(dmd):
global_catalog = dmd.zport.global_catalog
uncat = transact(global_catalog.uncatalog_object)
for brain in global_catalog():
try:
obj = brain.getObject()
except Exception:
log.warn("Found unresolvable path, deleting: %s", brain.getPath())
uncat(brain.getPath())
log.info("Finished scanning catalog")
def _applyDataMap(self, device, datamap, commit=True):
"""Apply datamap to device. Return True if datamap changed device.
The default value for commit is True for backwards-compatibility
reasons. If you're a new caller to ApplyDataMap._applyData you should
probably set commit to False and handle your own transactions.
"""
self.zing_datamap_handler.add_datamap(device, datamap)
if commit:
result = transact(self._applyDataMapImpl)(device, datamap)
else:
result = self._applyDataMapImpl(device, datamap)
return result
def _applyDataMap(self, device, datamap, commit=True):
"""Apply datamap to device. Return True if datamap changed device.
The default value for commit is True for backwards-compatibility
reasons. If you're a new caller to ApplyDataMap._applyData you should
probably set commit to False and handle your own transactions.
"""
self.zing_datamap_handler.add_datamap(device, datamap)
if commit:
result = transact(self._applyDataMapImpl)(device, datamap)
else:
result = self._applyDataMapImpl(device, datamap)
return result
def _cleanCatalog(self, zport):
global_catalog = self._getCatalog(zport)
uncat = transact(global_catalog.uncatalog_object)
for brain in global_catalog():
try:
obj = brain.getObject()
except Exception:
log.warn("Found unresolvable path, deleting: %s", brain.getPath())
uncat(brain.getPath())
log.info("Finished scanning catalog")
return defer.succeed(None)
def _clean_catalog(self):
drop_all_arguments()
zc = ZenCatalog()
dmd = zc.dmd
zport = dmd.zport
global_catalog = dmd.global_catalog
uncat = transact(global_catalog.uncatalog_object)
for brain in global_catalog():
try:
obj = brain.getObject()
except KeyError:
log.warn("Found unresolvable path, deleting: %s", brain.getPath())
uncat(brain.getPath())
except Exception:
log.warn("Unexpected exception loading object %s. Skipping!", brain.getPath(), exc_info=True)
log.info("Finished scanning catalog")
def commit_in_batches(zc, buffer_consumer, inbox, buffer_size, counter=None):
_commit_buffer = transact(buffer_consumer)
def commit_buffer(buf):
while True:
try:
return _commit_buffer(buf)
except (AttributeError, ClientDisconnected, DisconnectedError):
_reconnect(zc)
continue
buffer = []
try:
while True:
try:
element = inbox.get_nowait()
except Empty:
check_for_dead_parent()
time.sleep(0.1)
continue
if element is None:
# End of inbox. We're done here.
break
buffer.append(element)
if len(buffer) >= buffer_size:
delta = commit_buffer(buffer)
if counter:
if delta is None:
delta = len(buffer)
counter.increment(delta)
buffer = []
finally:
try:
delta = commit_buffer(buffer)
except BaseException:
log.warn("Failed to commit some changes.", exc_info=True)
for element in buffer:
put_or_die(inbox, element)
else:
if counter:
if delta is None:
delta = len(buffer)
counter.increment(delta)
def commit_in_batches(zc, buffer_consumer, inbox, buffer_size, counter=None):
_commit_buffer = transact(buffer_consumer)
def commit_buffer(buf):
while True:
try:
return _commit_buffer(buf)
except (AttributeError, ClientDisconnected, DisconnectedError):
_reconnect(zc)
continue
buffer = []
try:
while True:
try:
element = inbox.get_nowait()
except Empty:
check_for_dead_parent()
time.sleep(0.1)
continue
if element is None:
# End of inbox. We're done here.
break
buffer.append(element)
if len(buffer) >= buffer_size:
delta = commit_buffer(buffer)
if counter:
if delta is None:
delta = len(buffer)
counter.increment(delta)
buffer = []
finally:
try:
delta = commit_buffer(buffer)
except BaseException:
log.warn("Failed to commit some changes.", exc_info=True)
for element in buffer:
put_or_die(inbox, element)
else:
if counter:
if delta is None:
delta = len(buffer)
counter.increment(delta)
def _clean_catalog(self):
drop_all_arguments()
zc = ZenCatalog()
dmd = zc.dmd
zport = dmd.zport
global_catalog = dmd.global_catalog
uncat = transact(global_catalog.uncatalog_object)
for brain in global_catalog():
try:
obj = brain.getObject()
except KeyError:
log.warn("Found unresolvable path, deleting: %s",
brain.getPath())
uncat(brain.getPath())
except Exception:
log.warn("Unexpected exception loading object %s. Skipping!",
brain.getPath(),
exc_info=True)
log.info("Finished scanning catalog")
def quickstartWizardFinished(self):
# a place to hook up anything that needs to happen
app = self.context.dmd.primaryAq().getParentNode().getParentNode()
transact(notify)(QuickstartWizardFinishedEvent(app))
return DirectResponse.succeed()
def setProdState(self, state, ending=False, batchSize=None,
inTransaction=False):
"""
At any one time there is one production state for each device to be in,
and that is the state that is the most 'blacked out' in all of the active
maintenance windows affecting that device. When the last maintenance
window affecting a device has ended, the original production state of the
device is used to determine the end state of the device.
Maintenance windows are processed by zenjobs in batch so the ordering
of when two maintenance windows that end at the same time get processed
is non-deterministic. Since there is only one stop production state now,
this is not an issue.
@parameter state: hint from the maint window about device's start or stop state
@type state: integer
@parameter ending: are we ending a maintenance window?
@type ending: boolean
@parameter batchSize: number of processed devices per separate transaction
@type batchSize: integer
@parameter inTransaction: process each batch in separate transaction
@type inTransaction: boolean
"""
# Note: self.begin() starts our window before we get called, so the
# following takes into account our window state too.
# Conversely, self.end() ends the window before calling this code.
devices = self.fetchDevices()
minDevProdStates = self.fetchDeviceMinProdStates(devices)
def _setProdState(devices_batch):
for device in devices_batch:
# In case we try to move Component Group into MW
# some of objects in CG may not have production state
# we skip them.
if ending:
# Note: If no maintenance windows apply to a device, then the
# device won't exist in minDevProdStates
# This takes care of the case where there are still active
# maintenance windows.
minProdState = minDevProdStates.get(device.id,
device.getPreMWProductionState())
elif device.id in minDevProdStates:
minProdState = minDevProdStates[device.id]
else: # This is impossible for us to ever get here as minDevProdStates
# has been added by self.fetchDeviceMinProdStates()
log.error("The device %s does not appear in any maintenance"
" windows (including %s -- which is just starting).",
device.id, self.displayName())
continue
# ZEN-13197: skip decommissioned devices
if device.getPreMWProductionState() and device.getPreMWProductionState() < 300:
continue
self._p_changed = 1
# Changes the current state for a device, but *not*
# the preMWProductionState
oldProductionState = self.dmd.convertProdState(device.getProductionState())
# When MW ends Components will acquire production state from device
if not minProdState:
newProductionState = "Acquired from parent"
else:
newProductionState = self.dmd.convertProdState(minProdState)
log.info("MW %s changes %s's production state from %s to %s",
self.displayName(), device.id, oldProductionState,
newProductionState)
audit('System.Device.Edit', device, starting=str(not ending),
maintenanceWindow=self.displayName(),
productionState=newProductionState,
oldData_={'productionState':oldProductionState})
if minProdState is None:
device.resetProductionState()
else:
device.setProdState(minProdState, maintWindowChange=True)
if inTransaction:
processFunc = transact(_setProdState)
# Commit transaction as errors during batch processing may
# abort transaction and changes to the object will not be saved.
transaction.commit()
else:
processFunc = _setProdState
if batchSize:
for i in xrange(0, len(devices), batchSize):
log.info('MW %s processing batch #%s', self.displayName(),
i / batchSize + 1)
processFunc(devices[i:i + batchSize])
else:
processFunc(devices)
def _setSnmpLastCollection(self, device):
transactional = transact(device.setSnmpLastCollection)
return self._do_with_retries(transactional)
def catalog_the_things(worker_id, inbox, trash, buffer_size, permissions_only, print_progress=None):
# We don't import this above, because it didn't exist prior to 4.2.
# It's safe to import here, because we only get here if the CatalogService
# is installed, which didn't exist until 4.2.
from Products.Zuul.catalog.global_catalog import initializeGlobalCatalog
ignore_interruptions()
drop_all_arguments()
zc = ZenCatalog()
dmd = zc.dmd
zport = dmd.getPhysicalRoot().zport
global_catalog = _get_global_catalog(zport)
catalog_id = local_catalog_id(worker_id)
def _create_local_catalog():
local_catalog = global_catalog.__class__()
initializeGlobalCatalog(local_catalog)
try:
zport._delObject(catalog_id)
except:
pass
zport._setObject(catalog_id, local_catalog)
create_local_catalog = transact(_create_local_catalog, retries=100)
create_local_catalog()
local_catalog = getattr(zport, catalog_id, None)
catalog = local_catalog._catalog
counter = ProgressCounter(print_progress=print_progress)
def _catalog_one(primary_path):
try:
obj = dmd.unrestrictedTraverse(primary_path)
except (AttributeError, ClientDisconnected, DisconnectedError):
raise
except KeyError:
log.debug("Could not load object: %s", primary_path)
put_or_die(trash, primary_path)
return False
except Exception:
log.warn("Unexpected exception loading object %s. Skipping!", primary_path, exc_info=True)
return False
if obj is None:
log.debug("%s does not exist", primary_path)
put_or_die(trash, primary_path)
return False
try:
uid = global_catalog._catalog.uids.get(primary_path, None)
if uid is not None:
catalog.uids[primary_path] = uid
catalog.paths[uid] = global_catalog._catalog.paths.get(uid, None)
catalog.data[uid] = global_catalog._catalog.data.get(uid, None)
if permissions_only:
catalog.catalog_object(obj, update_metadata=False,
idxs=("allowedRolesAndUsers",))
else:
# We intentionally don't do legacy indexing:
# if hasattr(obj, 'index_object'): obj.index_object()
catalog.catalog_object(obj)
except (AttributeError, ClientDisconnected, DisconnectedError):
raise
except Exception:
log.info("Error cataloging object %s. Skipping", primary_path,
exc_info=(log.isEnabledFor(logging.DEBUG)))
return False
return True
def _catalog_several(primary_paths):
dmd._p_jar.sync()
count = 0
for primary_path in primary_paths:
if _catalog_one(primary_path):
count += 1
return count
commit_in_batches(zc, _catalog_several, inbox, buffer_size, counter)
def scan_catalog(catalogObject, fix, dmd, log, createEvents):
"""Scan through a catalog looking for broken references"""
# Fix for ZEN-14717 (only for global_catalog)
if (catalogObject.prettyName == 'global_catalog'):
global_catalog_paths_to_uids(catalogObject, fix, dmd, log, createEvents)
catalog = eval(catalogObject.dmdPath)
catalogObject.initialSize = len(catalog)
print("[%s] Examining %-35s (%d Objects)" %
(time.strftime("%Y-%m-%d %H:%M:%S"), catalogObject.prettyName, catalogObject.initialSize))
log.info("Examining %s catalog with %d objects" % (catalogObject.prettyName, catalogObject.initialSize))
currentCycle = 0
while (currentCycle < maxCycles):
currentCycle += 1
catalogObject.runResults[currentCycle] = {'itemCount': ZenToolboxUtils.Counter(0),
'errorCount': ZenToolboxUtils.Counter(0),
'repairCount': ZenToolboxUtils.Counter(0)
}
log.info("Beginning cycle %d for catalog %s" % (currentCycle, catalogObject.prettyName))
scan_progress_message(False, fix, currentCycle, catalogObject.prettyName, 0, 0, log)
try:
brains = eval(catalogObject.dmdPath)()
except Exception:
raise
catalogSize = len(brains)
if (catalogSize > 50):
progressBarChunkSize = (catalogSize//50) + 1
else:
progressBarChunkSize = 1
for brain in brains:
catalogObject.runResults[currentCycle]['itemCount'].increment()
if (catalogObject.runResults[currentCycle]['itemCount'].value() % progressBarChunkSize) == 0:
chunkNumber = catalogObject.runResults[currentCycle]['itemCount'].value() // progressBarChunkSize
scan_progress_message(False, fix, currentCycle, catalogObject.prettyName,
catalogObject.runResults[currentCycle]['errorCount'].value(), chunkNumber, log)
try:
testReference = brain.getObject()
testReference._p_deactivate()
except Exception:
catalogObject.runResults[currentCycle]['errorCount'].increment()
objectPathString = brain.getPath()
log.error("Catalog %s contains broken object %s" % (catalogObject.prettyName, objectPathString))
if fix:
log.info("Attempting to uncatalog %s" % (objectPathString))
try:
catalogObject.runResults[currentCycle]['repairCount'].increment()
transact(catalog.uncatalog_object)(objectPathString)
except Exception as e:
log.exception(e)
# Final transaction.abort() to try and free up used memory
log.debug("Calling transaction.abort() to minimize memory footprint")
transaction.abort()
scan_progress_message(True, fix, currentCycle, catalogObject.prettyName,
catalogObject.runResults[currentCycle]['errorCount'].value(), chunkNumber, log)
if fix:
if catalogObject.runResults[currentCycle]['errorCount'].value() == 0:
break
if currentCycle > 1:
if catalogObject.runResults[currentCycle]['errorCount'].value() == catalogObject.runResults[currentCycle-1]['errorCount'].value():
break
if createEvents:
scriptName = os.path.basename(__file__).split('.')[0]
eventMsg = ""
for cycleID in catalogObject.runResults.keys():
eventMsg += "Cycle %d scanned %d items, found %d errors and attempted %d repairs\n" % \
(cycleID, catalogObject.runResults[cycleID]['itemCount'].value(),
catalogObject.runResults[cycleID]['errorCount'].value(),
catalogObject.runResults[cycleID]['repairCount'].value())
if not catalogObject.runResults[currentCycle]['errorCount'].value():
eventSeverity = 1
if currentCycle == 1:
eventSummaryMsg = "'%s' - No Errors Detected (%d total items)" % \
(catalogObject.prettyName, catalogObject.initialSize)
else:
eventSummaryMsg = "'%s' - No Errors Detected [--fix was successful] (%d total items)" % \
(catalogObject.prettyName, catalogObject.initialSize)
else:
eventSeverity = 4
if fix:
eventSummaryMsg = "'%s' - %d Errors Remain after --fix [consult log file] (%d total items)" % \
(catalogObject.prettyName, catalogObject.runResults[currentCycle]['errorCount'].value(), catalogObject.initialSize)
else:
eventSummaryMsg = "'%s' - %d Errors Detected [run with --fix] (%d total items)" % \
(catalogObject.prettyName, catalogObject.runResults[currentCycle]['errorCount'].value(), catalogObject.initialSize)
log.debug("Creating event with %s, %s" % (eventSummaryMsg, eventSeverity))
ZenToolboxUtils.send_summary_event(
eventSummaryMsg, eventSeverity,
scriptName, catalogObject.prettyName,
documentationURL, dmd, eventMsg
)
return (catalogObject.runResults[currentCycle]['errorCount'].value() != 0)
def scan_catalog(catalog_name, catalog_list, fix, max_cycles, dmd, log):
"""Scan through a catalog looking for broken references"""
catalog = catalog_list[0]
initial_catalog_size = catalog_list[1]
print("[%s] Examining %-35s (%d Objects)" %
(time.strftime("%Y-%m-%d %H:%M:%S"), catalog_name,
initial_catalog_size))
log.info("Examining %s catalog with %d objects" %
(catalog_name, initial_catalog_size))
number_of_issues = -1
current_cycle = 0
if not fix:
max_cycles = 1
while ((current_cycle < max_cycles) and (number_of_issues != 0)):
number_of_issues = 0
current_cycle += 1
if (fix):
log.info("Beginning cycle %d for catalog %s" %
(current_cycle, catalog_name))
scanned_count = 0
progress_bar_chunk_size = 1
# ZEN-12165: show progress bar immediately before 'for' time overhead, before loading catalog
scan_progress_message(False, fix, current_cycle, catalog_name, 0, 0,
log)
try:
brains = catalog()
catalog_size = len(brains)
if (catalog_size > 50):
progress_bar_chunk_size = (catalog_size // 50) + 1
except Exception:
raise
for brain in brains:
scanned_count += 1
if (scanned_count % progress_bar_chunk_size) == 0:
chunk_number = scanned_count // progress_bar_chunk_size
scan_progress_message(False, fix, current_cycle, catalog_name,
number_of_issues, chunk_number, log)
try:
test_reference = brain.getObject()
test_reference._p_deactivate()
except Exception:
number_of_issues += 1
object_path_string = brain.getPath()
log.error("Catalog %s contains broken object %s" %
(catalog_name, object_path_string))
if fix:
log.info("Attempting to uncatalog %s" %
(object_path_string))
try:
transact(catalog.uncatalog_object)(object_path_string)
except Exception as e:
log.exception(e)
# Final transaction.abort() to try and free up used memory
log.debug("Calling transaction.abort() to minimize memory footprint")
transaction.abort()
scan_progress_message(True, fix, current_cycle, catalog_name,
number_of_issues, chunk_number, log)
if number_of_issues > 0:
return True
return False
def catalog_the_things(worker_id,
inbox,
trash,
buffer_size,
permissions_only,
print_progress=None):
# We don't import this above, because it didn't exist prior to 4.2.
# It's safe to import here, because we only get here if the CatalogService
# is installed, which didn't exist until 4.2.
from Products.Zuul.catalog.global_catalog import initializeGlobalCatalog
ignore_interruptions()
drop_all_arguments()
zc = ZenCatalog()
dmd = zc.dmd
zport = dmd.getPhysicalRoot().zport
global_catalog = _get_global_catalog(zport)
catalog_id = local_catalog_id(worker_id)
def _create_local_catalog():
local_catalog = global_catalog.__class__()
initializeGlobalCatalog(local_catalog)
try:
zport._delObject(catalog_id)
except:
pass
zport._setObject(catalog_id, local_catalog)
create_local_catalog = transact(_create_local_catalog, retries=100)
create_local_catalog()
local_catalog = getattr(zport, catalog_id, None)
catalog = local_catalog._catalog
counter = ProgressCounter(print_progress=print_progress)
def _catalog_one(primary_path):
try:
obj = dmd.unrestrictedTraverse(primary_path)
except (AttributeError, ClientDisconnected, DisconnectedError):
raise
except KeyError:
log.debug("Could not load object: %s", primary_path)
put_or_die(trash, primary_path)
return False
except Exception:
log.warn("Unexpected exception loading object %s. Skipping!",
primary_path,
exc_info=True)
return False
if obj is None:
log.debug("%s does not exist", primary_path)
put_or_die(trash, primary_path)
return False
try:
uid = global_catalog._catalog.uids.get(primary_path, None)
if uid is not None:
catalog.uids[primary_path] = uid
catalog.paths[uid] = global_catalog._catalog.paths.get(
uid, None)
catalog.data[uid] = global_catalog._catalog.data.get(uid, None)
if permissions_only:
catalog.catalog_object(obj,
update_metadata=False,
idxs=("allowedRolesAndUsers", ))
else:
# We intentionally don't do legacy indexing:
# if hasattr(obj, 'index_object'): obj.index_object()
catalog.catalog_object(obj)
except (AttributeError, ClientDisconnected, DisconnectedError):
raise
except Exception:
log.info("Error cataloging object %s. Skipping",
primary_path,
exc_info=(log.isEnabledFor(logging.DEBUG)))
return False
return True
def _catalog_several(primary_paths):
dmd._p_jar.sync()
count = 0
for primary_path in primary_paths:
if _catalog_one(primary_path):
count += 1
return count
commit_in_batches(zc, _catalog_several, inbox, buffer_size, counter)
def wrapper(obj, *args, **kwargs):
if obj.options.nocommit:
return f.__call__(obj, *args, **kwargs)
else:
return transact(f).__call__(obj, *args, **kwargs)
def transactional(f):
return f if self.options.nocommit else transact(f)
def transactional(f):
return f if self.options.nocommit else transact(f)
def scan_catalog(catalog_name, catalog_list, fix, max_cycles, dmd, log, create_events):
"""Scan through a catalog looking for broken references"""
# Fix for ZEN-14717 (only for global_catalog)
if (catalog_name == 'global_catalog'):
global_catalog_rids(catalog_name, catalog_list, fix, max_cycles, dmd, log, create_events)
catalog = catalog_list[0]
initial_catalog_size = catalog_list[1]
number_of_issues = -1
current_cycle = 0
if not fix:
max_cycles = 1
print("[%s] Examining %-35s (%d Objects)" %
(time.strftime("%Y-%m-%d %H:%M:%S"), catalog_name, initial_catalog_size))
log.info("Examining %s catalog with %d objects" % (catalog_name, initial_catalog_size))
while ((current_cycle < max_cycles) and (number_of_issues != 0)):
number_of_issues = 0
current_cycle += 1
if (fix):
log.info("Beginning cycle %d for catalog %s" % (current_cycle, catalog_name))
scanned_count = 0
progress_bar_chunk_size = 1
# ZEN-12165: show progress bar immediately before 'for' time overhead, before loading catalog
scan_progress_message(False, fix, current_cycle, catalog_name, 0, 0, log)
try:
brains = catalog()
catalog_size = len(brains)
if (catalog_size > 50):
progress_bar_chunk_size = (catalog_size//50) + 1
except Exception:
raise
for brain in brains:
scanned_count += 1
if (scanned_count % progress_bar_chunk_size) == 0:
chunk_number = scanned_count // progress_bar_chunk_size
scan_progress_message(False, fix, current_cycle, catalog_name, number_of_issues, chunk_number, log)
try:
test_reference = brain.getObject()
test_reference._p_deactivate()
except Exception:
number_of_issues += 1
object_path_string = brain.getPath()
log.error("Catalog %s contains broken object %s" % (catalog_name, object_path_string))
if fix:
log.info("Attempting to uncatalog %s" % (object_path_string))
try:
transact(catalog.uncatalog_object)(object_path_string)
except Exception as e:
log.exception(e)
# Final transaction.abort() to try and free up used memory
log.debug("Calling transaction.abort() to minimize memory footprint")
transaction.abort()
scan_progress_message(True, fix, current_cycle, catalog_name, number_of_issues, chunk_number, log)
if create_events:
if number_of_issues > 0:
eventSummaryMsg = "'%s' - %d Error(s) Detected (%d total items)" % (catalog_name, number_of_issues, initial_catalog_size)
eventSeverity = 4
else:
eventSummaryMsg = "'%s' - No Errors Detected (%d total items)" % (catalog_name, initial_catalog_size)
eventSeverity = 1
dmd.ZenEventManager.sendEvent({
'device' : 'localhost',
'summary' : eventSummaryMsg,
'message' : eventSummaryMsg,
'component' : 'zenoss_toolbox',
'severity' : eventSeverity,
'eventClass' : '/Status',
'eventKey' : "%s" % (catalog_name),
'dedupid' : "zenoss_toolbox_zencatalogscan.%s" % (catalog_name),
'eventClassKey' : "zenoss_toolbox_zencatalogscan",
'details' : "Consult https://support.zenoss.com/hc/en-us/articles/203118075 for additional information"
})
return (number_of_issues != 0)
def _setSnmpLastCollection(self, device):
transactional = transact(device.setSnmpLastCollection)
return self._do_with_retries(transactional)
def quickstartWizardFinished(self):
# a place to hook up anything that needs to happen
app = self.context.dmd.primaryAq().getParentNode().getParentNode()
transact(notify)(QuickstartWizardFinishedEvent(app))
return DirectResponse.succeed()
def wrapper(obj, *args, **kwargs):
if obj.options.nocommit:
return f.__call__(obj, *args, **kwargs)
else:
return transact(f).__call__(obj, *args, **kwargs)
def scan_catalog(catalogObject, fix, dmd, log, createEvents):
"""Scan through a catalog looking for broken references"""
# Fix for ZEN-14717 (only for global_catalog)
if (catalogObject.prettyName == 'global_catalog'):
global_catalog_paths_to_uids(catalogObject, fix, dmd, log,
createEvents)
catalog = eval(catalogObject.dmdPath)
catalogObject.initialSize = len(catalog)
print("[%s] Examining %-35s (%d Objects)" %
(time.strftime("%Y-%m-%d %H:%M:%S"), catalogObject.prettyName,
catalogObject.initialSize))
log.info("Examining %s catalog with %d objects" %
(catalogObject.prettyName, catalogObject.initialSize))
currentCycle = 0
while (currentCycle < maxCycles):
currentCycle += 1
catalogObject.runResults[currentCycle] = {
'itemCount': ZenToolboxUtils.Counter(0),
'errorCount': ZenToolboxUtils.Counter(0),
'repairCount': ZenToolboxUtils.Counter(0)
}
log.info("Beginning cycle %d for catalog %s" %
(currentCycle, catalogObject.prettyName))
scan_progress_message(False, fix, currentCycle,
catalogObject.prettyName, 0, 0, log)
try:
brains = eval(catalogObject.dmdPath)()
except Exception:
raise
catalogSize = len(brains)
if (catalogSize > 50):
progressBarChunkSize = (catalogSize // 50) + 1
else:
progressBarChunkSize = 1
for brain in brains:
catalogObject.runResults[currentCycle]['itemCount'].increment()
if (catalogObject.runResults[currentCycle]['itemCount'].value() %
progressBarChunkSize) == 0:
chunkNumber = catalogObject.runResults[currentCycle][
'itemCount'].value() // progressBarChunkSize
scan_progress_message(
False, fix, currentCycle, catalogObject.prettyName,
catalogObject.runResults[currentCycle]
['errorCount'].value(), chunkNumber, log)
try:
testReference = brain.getObject()
testReference._p_deactivate()
except Exception:
catalogObject.runResults[currentCycle]['errorCount'].increment(
)
objectPathString = brain.getPath()
log.error("Catalog %s contains broken object %s" %
(catalogObject.prettyName, objectPathString))
if fix:
log.info("Attempting to uncatalog %s" % (objectPathString))
try:
catalogObject.runResults[currentCycle][
'repairCount'].increment()
transact(catalog.uncatalog_object)(objectPathString)
except Exception as e:
log.exception(e)
# Final transaction.abort() to try and free up used memory
log.debug("Calling transaction.abort() to minimize memory footprint")
transaction.abort()
scan_progress_message(
True, fix, currentCycle, catalogObject.prettyName,
catalogObject.runResults[currentCycle]['errorCount'].value(),
chunkNumber, log)
if fix:
if catalogObject.runResults[currentCycle]['errorCount'].value(
) == 0:
break
if currentCycle > 1:
if catalogObject.runResults[currentCycle]['errorCount'].value(
) == catalogObject.runResults[currentCycle -
1]['errorCount'].value():
break
if createEvents:
scriptName = os.path.basename(__file__).split('.')[0]
eventMsg = ""
for cycleID in catalogObject.runResults.keys():
eventMsg += "Cycle %d scanned %d items, found %d errors and attempted %d repairs\n" % \
(cycleID, catalogObject.runResults[cycleID]['itemCount'].value(),
catalogObject.runResults[cycleID]['errorCount'].value(),
catalogObject.runResults[cycleID]['repairCount'].value())
if not catalogObject.runResults[currentCycle]['errorCount'].value():
eventSeverity = 1
if currentCycle == 1:
eventSummaryMsg = "'%s' - No Errors Detected (%d total items)" % \
(catalogObject.prettyName, catalogObject.initialSize)
else:
eventSummaryMsg = "'%s' - No Errors Detected [--fix was successful] (%d total items)" % \
(catalogObject.prettyName, catalogObject.initialSize)
else:
eventSeverity = 4
if fix:
eventSummaryMsg = "'%s' - %d Errors Remain after --fix [consult log file] (%d total items)" % \
(catalogObject.prettyName, catalogObject.runResults[currentCycle]['errorCount'].value(), catalogObject.initialSize)
else:
eventSummaryMsg = "'%s' - %d Errors Detected [run with --fix] (%d total items)" % \
(catalogObject.prettyName, catalogObject.runResults[currentCycle]['errorCount'].value(), catalogObject.initialSize)
log.debug("Creating event with %s, %s" %
(eventSummaryMsg, eventSeverity))
ZenToolboxUtils.send_summary_event(eventSummaryMsg, eventSeverity,
scriptName,
catalogObject.prettyName,
documentationURL, dmd, eventMsg)
return (catalogObject.runResults[currentCycle]['errorCount'].value() != 0)
def setProdState(self,
state,
ending=False,
batchSize=None,
inTransaction=False):
"""
At any one time there is one production state for each device to be in,
and that is the state that is the most 'blacked out' in all of the active
maintenance windows affecting that device. When the last maintenance
window affecting a device has ended, the original production state of the
device is used to determine the end state of the device.
Maintenance windows are processed by zenjobs in batch so the ordering
of when two maintenance windows that end at the same time get processed
is non-deterministic. Since there is only one stop production state now,
this is not an issue.
@parameter state: hint from the maint window about device's start or stop state
@type state: integer
@parameter ending: are we ending a maintenance window?
@type ending: boolean
@parameter batchSize: number of processed devices per separate transaction
@type batchSize: integer
@parameter inTransaction: process each batch in separate transaction
@type inTransaction: boolean
"""
# Note: self.begin() starts our window before we get called, so the
# following takes into account our window state too.
# Conversely, self.end() ends the window before calling this code.
devices = self.fetchDevices()
unchangedDevices = []
minDevProdStates = self.fetchDeviceMinProdStates(devices)
def _setProdState(devices_batch):
for device in devices_batch:
guid = IGlobalIdentifier(device).getGUID()
# In case we try to move Component Group into MW
# some of objects in CG may not have production state
# we skip them.
if ending:
# Note: If no maintenance windows apply to a device, then the
# device won't exist in minDevProdStates
# This takes care of the case where there are still active
# maintenance windows.
minProdState = minDevProdStates.get(
guid, device.getPreMWProductionState())
elif guid in minDevProdStates:
minProdState = minDevProdStates[guid]
else: # This is impossible for us to ever get here as minDevProdStates
# has been added by self.fetchDeviceMinProdStates()
log.error(
"The device %s does not appear in any maintenance"
" windows (including %s -- which is just starting).",
device.id, self.displayName())
continue
# ZEN-13197: skip decommissioned devices
if device.getPreMWProductionState(
) and device.getPreMWProductionState() < 300:
continue
self._p_changed = 1
# Changes the current state for a device, but *not*
# the preMWProductionState
oldProductionState = self.dmd.convertProdState(
device.getProductionState())
# When MW ends Components will acquire production state from device
if not minProdState:
newProductionState = "Acquired from parent"
else:
newProductionState = self.dmd.convertProdState(
minProdState)
log.info("MW %s changes %s's production state from %s to %s",
self.displayName(), device.id, oldProductionState,
newProductionState)
audit('System.Device.Edit',
device,
starting=str(not ending),
maintenanceWindow=self.displayName(),
productionState=newProductionState,
oldData_={'productionState': oldProductionState})
if minProdState is None:
device.resetProductionState()
else:
device.setProdState(minProdState, maintWindowChange=True)
def retrySingleDevices(devices_batch):
log.warn("Retrying devices individually")
for dev in devices_batch:
try:
processFunc([dev])
except Exception:
log.exception(
"The production stage change for %s raised the exception.",
dev)
unchangedDevices.append(dev)
def processBatchOfDevices(devices, batchSize):
for i in xrange(0, len(devices), batchSize):
log.info('MW %s processing batch #%s', self.displayName(),
i / batchSize + 1)
dev_chunk = devices[i:i + batchSize]
try:
processFunc(dev_chunk)
except (ConflictError, POSKeyError, ReadConflictError) as e:
log.warn(
"While processing batch of %d devices exception was raised. %s",
len(dev_chunk), e)
retrySingleDevices(dev_chunk)
except Exception:
# We're expecting ConflictError, POSKeyError, ReadConflictError, and handle them with retries
# production state change. All other exceptions are an unexplored area that should be properly
# processed in the future instead of the stub below.
log.exception("Unexpected Exception encountered")
retrySingleDevices(dev_chunk)
if inTransaction:
processFunc = transact(_setProdState)
# Commit transaction as errors during batch processing may
# abort transaction and changes to the object will not be saved.
transaction.commit()
else:
processFunc = _setProdState
# Adding exception handling for the following:
# ConflictError, POSKeyError and ReadConflictError.
#
# If any of the listed exceptions are encountered, we will retry
# devices in the batch individually. If batchSize isn't
# specified and we encounter one of these exceptions, we will
# then specify a batch size and retry the devices in batches.
# If there are exceptions in these batches, we will retry the
# devices individually. This will ensure that maintenance
# windows as a whole do not fail due to these exceptions.
# Fixes ZEN-31805.
if batchSize:
processBatchOfDevices(devices, batchSize)
else:
try:
processFunc(devices)
except (ConflictError, POSKeyError, ReadConflictError) as e:
log.warn(
"Exception encountered and no batchSize specified. "
"%s. Retrying in batches.", e)
processBatchOfDevices(devices, 10)
except Exception:
log.exception("Unexpected Exception encountered")
processBatchOfDevices(devices, 10)
if unchangedDevices:
log.error("Unable to change Production State on: %s",
unchangedDevices)
