def migrate(self): """ Determine the correct migrate steps to run and apply them """ steps = self.determineSteps() if steps: for m in steps: m.prepare() currentDbVers = self._currentVersion() if steps[-1].version > currentDbVers and not self.options.dont_bump: self.message('Database going to version %s' % steps[-1].version.long()) # hide uncatalog error messages since they do not do any harm log = logging.getLogger('Zope.ZCatalog') oldLevel = log.getEffectiveLevel() log.setLevel(HIGHER_THAN_CRITICAL) for m in steps: self.message('Installing %s (%s)' % (m.name(), m.version.short())) m.cutover(self.dmd) if m.version > currentDbVers and not self.options.dont_bump: self.dmd.version = m.version.long() for m in steps: m.cleanup() log.setLevel(oldLevel) cleanup() if not self.options.steps: self.message('Loading Reports') rl = ReportLoader(noopts=True, app=self.app) # when reports change make sure the new version is loaded during the migrate rl.options.force = True rl.options.logseverity = self.options.logseverity + 10 rl.setupLogging() rl.loadDatabase() # Update JavaScript portlets self.dmd.ZenPortletManager.update_source()
def migrate(self): """ Determine the correct migrate steps to run and apply them """ steps = self.determineSteps() if steps: for m in steps: m.prepare() currentDbVers = self._currentVersion() if steps[-1].version > currentDbVers: self.message('Database going to version %s' % steps[-1].version.long()) # hide uncatalog error messages since they do not do any harm log = logging.getLogger('Zope.ZCatalog') oldLevel = log.getEffectiveLevel() log.setLevel(HIGHER_THAN_CRITICAL) for m in steps: self.message('Installing %s (%s)' % (m.name(), m.version.short())) m.cutover(self.dmd) if m.version > currentDbVers: self.dmd.version = m.version.long() for m in steps: m.cleanup() log.setLevel(oldLevel) cleanup() if not self.options.steps: self.message('Loading Reports') rl = ReportLoader(noopts=True, app=self.app) # when reports change make sure the new version is loaded during the migrate rl.options.force = True rl.options.logseverity = self.options.logseverity + 10 rl.setupLogging() rl.loadDatabase() # Update JavaScript portlets self.dmd.ZenPortletManager.update_source()
def loadFile(self, root, id, fullname): rpt = ReportLoader.loadFile(self, root, id, fullname) rpt.addRelation('pack', pack) return rpt
def setUp(self): self.rp_load = ReportLoader()
class ReportLoaderTest(BaseTestCase): def setUp(self): self.rp_load = ReportLoader() def test_loadDatabase(self): self.rp_load.loadAllReports = create_autospec( self.rp_load.loadAllReports) self.rp_load.loadDatabase() self.rp_load.loadAllReports.assert_called_once_with() @patch('Products.ZenReports.ReportLoader.transaction.commit', autospec=True, spec_set=True) def test_loadAllReports(self, commit): repdir = zenPath('Products/ZenReports', self.rp_load.options.dir) self.rp_load.loadDirectory = create_autospec( self.rp_load.loadDirectory) self.rp_load.loadAllReports() self.rp_load.loadDirectory.assert_called_once_with(repdir) commit.assert_called_once_with() def test_loadAllReports_zp(self): self.rp_load.options.zenpack = True self.rp_load.getZenPackDirs = create_autospec( self.rp_load.getZenPackDirs) self.rp_load.loadAllReports() self.rp_load.getZenPackDirs.assert_called_once_with( self.rp_load.options.zenpack) def test_getZenPackDirs(self): zp_name = 'test_zp' zp_path = '/path/to/test_zp' zp_obj = Mock(id='test_zp') zp_obj.path = Mock(return_value=zp_path) self.rp_load.dmd.ZenPackManager.packs = create_autospec( self.rp_load.dmd.ZenPackManager.packs, return_value=[zp_obj]) self.rp_load.options.dir = 'reports' zp_dir_result = ['/path/to/test_zp/reports'] result = self.rp_load.getZenPackDirs(name=zp_name) self.assertEqual(result, zp_dir_result) self.assertIsInstance(result, list) self.assertEqual(len(result), 1) def test_getZenPackDirs_error(self): zp_name = 'noname_zp' zp_path = '/path/to/test_zp' zp_obj = Mock(id='test_zp') zp_obj.path = Mock(return_value=zp_path) self.rp_load.dmd.ZenPackManager.packs = create_autospec( self.rp_load.dmd.ZenPackManager.packs, return_value=[zp_obj]) self.rp_load.options.dir = 'reports' #set loglevel to 50(CRITICAL) this will remove error log self.rp_load.log.setLevel('CRITICAL') with self.assertRaises(SystemExit) as exc: self.rp_load.getZenPackDirs(name=zp_name) self.assertEqual(exc.exception.code, 1) @patch('Products.ZenReports.ReportLoader.os.walk', autospec=True, spec_set=True) def test_reports(self, walk): rp_dir = '/path/to/test_zp/reports/SomeReports' os_walk_data = [(rp_dir, [], ['reportName.rpt'])] walk.return_value = os_walk_data ret_data = [('/SomeReports', 'reportName', '/path/to/test_zp/reports/SomeReports/reportName.rpt')] result = self.rp_load.reports(rp_dir) self.assertEqual(result, ret_data) def test_unloadDirectory(self): rp_dir = '/path/to/test_zp/reports/SomeReports' orgpath = '/SomeReports' rp_id = 'reportName' report_data = [(orgpath, rp_id, '/path/to/test_zp/Reports/SomeReports/reportName.rpt')] rorg = Mock(id=rp_id) rorg_parent = Mock(id='Reports') setattr(rorg, rp_id, True) rorg._delObject = Mock() rorg.objectValues = Mock(return_value=False) rorg.getPrimaryParent = Mock(return_value=rorg_parent) self.rp_load.dmd.Reports.createOrganizer = Mock(return_value=rorg) self.rp_load.reports = Mock(return_value=report_data) self.rp_load.unloadDirectory(repdir=rp_dir) self.rp_load.dmd.Reports.createOrganizer.assert_called_once_with( orgpath) rorg._delObject.assert_called_with(rp_id) rorg.objectValues.assert_called_once_with() rorg.getPrimaryParent.assert_called_once_with() def test_unloadDirectory_false(self): '''test that _delObject method was not called ''' rp_dir = '/path/to/test_zp/reports/SomeReports' orgpath = '/SomeReports' rp_id = 'reportName' report_data = [(orgpath, rp_id, '/path/to/test_zp/Reports/SomeReports/reportName.rpt')] rorg = Mock(id='Reports') rorg_parent = Mock(id='Reports') rorg._delObject = Mock() rorg.objectValues = Mock(return_value=False) rorg.getPrimaryParent = Mock(return_value=rorg_parent) setattr(rorg, rp_id, False) self.rp_load.dmd.Reports.createOrganizer = Mock(return_value=rorg) self.rp_load.reports = Mock(return_value=report_data) self.rp_load.unloadDirectory(repdir=rp_dir) rorg._delObject.assert_not_called() def test_loadDirectory_force(self): full_path = '/path/to/test_zp/Reports/SomeReports/reportName.rpt' rp_dir = '/path/to/test_zp/reports/SomeReports' orgpath = '/SomeReports' rp_id = 'reportName' report_data = [(orgpath, rp_id, full_path)] self.rp_load.options.force = True rorg = Mock() report = Mock() #set that this report is not from zenpack report.pack = Mock(return_value=False) setattr(rorg, rp_id, report) rorg._delObject = Mock() rorg.objectValues = Mock(return_value=False) self.rp_load.dmd.Reports.createOrganizer = Mock(return_value=rorg) self.rp_load.reports = Mock(return_value=report_data) self.rp_load.loadFile = create_autospec(self.rp_load.loadFile) self.rp_load.loadDirectory(rp_dir) self.rp_load.dmd.Reports.createOrganizer.assert_called_once_with( orgpath) rorg._delObject.assert_called_once_with(rp_id) self.rp_load.loadFile.assert_called_with(rorg, rp_id, full_path) def test_loadDirectory(self): '''test that _delObject method was not called and we didn't overwrite reports ''' full_path = '/path/to/test_zp/Reports/SomeReports/reportName.rpt' rp_dir = '/path/to/test_zp/reports/SomeReports' orgpath = '/SomeReports' rp_id = 'reportName' report_data = [(orgpath, rp_id, full_path)] #force option is False by default, this is for better clarity self.rp_load.options.force = False rorg = Mock() setattr(rorg, rp_id, True) rorg._delObject = Mock() rorg.objectValues = Mock(return_value=False) self.rp_load.dmd.Reports.createOrganizer = Mock(return_value=rorg) self.rp_load.reports = Mock(return_value=report_data) self.rp_load.loadFile = Mock() self.rp_load.loadDirectory(rp_dir) rorg._delObject.assert_not_called() self.rp_load.loadFile.assert_not_called() @patch('__builtin__.file', autospec=True, spec_set=True) def test_loadFile(self, file_mock): rp_name = 'reportName' full_rp_path = '/path/to/test_zp/Reports/SomeReports/reportName.rpt' report_txt = "some report data" #mock build in file method and its instance read method file_read = Mock() file_read.read = Mock(return_value=report_txt) file_mock.return_value = file_read root = Mock() root._setObject = Mock() rp = self.rp_load.loadFile(root, rp_name, full_rp_path) self.assertIsInstance(rp, Report) self.assertEqual(rp.id, rp_name) root._setObject.assert_called_once_with(rp_name, rp)
def build(self): self.db = None self.storage = None conn = None try: self.zodbConnect() conn = self.db.open() root = conn.root() app = root.get('Application') if app and getattr(app, self.sitename, None) is not None: print "zport portal object exists; exiting." return except self.connectionFactory.exceptions.OperationalError as e: print "zenbuild: Database does not exist." sys.exit(1) finally: if conn: conn.close() if self.db: self.db.close() self.db = None if self.storage: self.storage.close() self.storage = None # TODO: remove the port condition, in here for now because there is no SQL dump of postgresql if self.options.fromXml: self.connect() from Products.ZenModel.ZentinelPortal import manage_addZentinelPortal manage_addZentinelPortal(self.app, self.sitename) site = self.app._getOb(self.sitename) # build index_html if self.app.hasObject('index_html'): self.app._delObject('index_html') from Products.PythonScripts.PythonScript import manage_addPythonScript manage_addPythonScript(self.app, 'index_html') newIndexHtml = self.app._getOb('index_html') text = 'container.REQUEST.RESPONSE.redirect(container.zport.virtualRoot() + "/zport/dmd/")\n' newIndexHtml.ZPythonScript_edit('', text) # build standard_error_message if self.app.hasObject('standard_error_message'): self.app._delObject('standard_error_message') file = open( zenPath('Products/ZenModel/dtml/standard_error_message.dtml')) try: text = file.read() finally: file.close() import OFS.DTMLMethod OFS.DTMLMethod.addDTMLMethod(self.app, id='standard_error_message', file=text) # Convert the acl_users folder at the root to a PAS folder and update # the login form to use the Zenoss login form Security.replaceACLWithPAS(self.app, deleteBackup=True) auth0_setup(self.app) account_locker_setup(self.app) # Add groupManager to zport.acl acl = site.acl_users if not hasattr(acl, 'groupManager'): plugins.ZODBGroupManager.addZODBGroupManager( acl, 'groupManager') acl.groupManager.manage_activateInterfaces([ 'IGroupsPlugin', ]) trans = transaction.get() trans.note("Initial ZentinelPortal load by zenbuild.py") trans.commit() print "ZentinelPortal loaded at %s" % self.sitename # build dmd from Products.ZenModel.DmdBuilder import DmdBuilder dmdBuilder = DmdBuilder(site, self.options.evthost, self.options.evtuser, self.options.evtpass, self.options.evtdb, self.options.evtport, self.options.smtphost, self.options.smtpport, self.options.pagecommand) dmdBuilder.build() transaction.commit() # Load XML Data from Products.ZenModel.XmlDataLoader import XmlDataLoader dl = XmlDataLoader(noopts=True, app=self.app) dl.loadDatabase() else: cmd = "gunzip -c %s | %s --usedb=zodb" % ( zenPath("Products/ZenModel/data/zodb.sql.gz"), zenPath("Products/ZenUtils/ZenDB.py"), ) returncode = os.system(cmd) if returncode: print >> sys.stderr, "There was a problem creating the database from the sql dump." sys.exit(1) # Relstorage may have already loaded items into the cache in the # initial connection to the database. We have to expire everything # in the cache in order to prevent errors with overlapping # transactions from the model which was just imported above. if self.options.zodb_cacheservers: self.flush_memcached(self.options.zodb_cacheservers.split()) self.connect() # Set all the attributes site = getattr(self.app, self.sitename, None) site.dmd.smtpHost = self.options.smtphost site.dmd.smtpPort = self.options.smtpport site.dmd.pageCommand = self.options.pagecommand site.dmd.uuid = None for evmgr in (site.dmd.ZenEventManager, site.dmd.ZenEventHistory): evmgr.username = self.options.evtuser evmgr.password = self.options.evtpass evmgr.database = self.options.evtdb evmgr.host = self.options.evthost evmgr.port = self.options.evtport transaction.commit() # Load reports from Products.ZenReports.ReportLoader import ReportLoader rl = ReportLoader(noopts=True, app=self.app) rl.loadDatabase()
class ReportLoaderTest(BaseTestCase): def setUp(self): self.rp_load = ReportLoader() def test_loadDatabase(self): self.rp_load.loadAllReports = create_autospec(self.rp_load.loadAllReports) self.rp_load.loadDatabase() self.rp_load.loadAllReports.assert_called_once_with() @patch( 'Products.ZenReports.ReportLoader.transaction.commit', autospec=True, spec_set=True ) def test_loadAllReports(self, commit): repdir = zenPath('Products/ZenReports', self.rp_load.options.dir) self.rp_load.loadDirectory = create_autospec(self.rp_load.loadDirectory) self.rp_load.loadAllReports() self.rp_load.loadDirectory.assert_called_once_with(repdir) commit.assert_called_once_with() def test_loadAllReports_zp(self): self.rp_load.options.zenpack = True self.rp_load.getZenPackDirs = create_autospec(self.rp_load.getZenPackDirs) self.rp_load.loadAllReports() self.rp_load.getZenPackDirs.assert_called_once_with(self.rp_load.options.zenpack) def test_getZenPackDirs(self): zp_name = 'test_zp' zp_path = '/path/to/test_zp' zp_obj = Mock(id='test_zp') zp_obj.path = Mock(return_value=zp_path) self.rp_load.dmd.ZenPackManager.packs = create_autospec( self.rp_load.dmd.ZenPackManager.packs, return_value=[zp_obj] ) self.rp_load.options.dir = 'reports' zp_dir_result = ['/path/to/test_zp/reports'] result = self.rp_load.getZenPackDirs(name=zp_name) self.assertEqual(result, zp_dir_result) self.assertIsInstance(result, list) self.assertEqual(len(result), 1) def test_getZenPackDirs_error(self): zp_name = 'noname_zp' zp_path = '/path/to/test_zp' zp_obj = Mock(id='test_zp') zp_obj.path = Mock(return_value=zp_path) self.rp_load.dmd.ZenPackManager.packs = create_autospec( self.rp_load.dmd.ZenPackManager.packs, return_value=[zp_obj] ) self.rp_load.options.dir = 'reports' #set loglevel to 50(CRITICAL) this will remove error log self.rp_load.log.setLevel('CRITICAL') with self.assertRaises(SystemExit) as exc: self.rp_load.getZenPackDirs(name=zp_name) self.assertEqual(exc.exception.code, 1) @patch( 'Products.ZenReports.ReportLoader.os.walk', autospec=True, spec_set=True ) def test_reports(self, walk): rp_dir = '/path/to/test_zp/reports/SomeReports' os_walk_data = [(rp_dir, [], ['reportName.rpt'])] walk.return_value = os_walk_data ret_data = [('/SomeReports', 'reportName', '/path/to/test_zp/reports/SomeReports/reportName.rpt' )] result = self.rp_load.reports(rp_dir) self.assertEqual(result, ret_data) def test_unloadDirectory(self): rp_dir = '/path/to/test_zp/reports/SomeReports' orgpath = '/SomeReports' rp_id = 'reportName' report_data = [(orgpath, rp_id, '/path/to/test_zp/Reports/SomeReports/reportName.rpt' )] rorg = Mock(id=rp_id) rorg_parent = Mock(id='Reports') setattr(rorg, rp_id, True) rorg._delObject = Mock() rorg.objectValues = Mock(return_value=False) rorg.getPrimaryParent = Mock(return_value=rorg_parent) self.rp_load.dmd.Reports.createOrganizer = Mock(return_value=rorg) self.rp_load.reports = Mock(return_value=report_data) self.rp_load.unloadDirectory(repdir=rp_dir) self.rp_load.dmd.Reports.createOrganizer.assert_called_once_with(orgpath) rorg._delObject.assert_called_with(rp_id) rorg.objectValues.assert_called_once_with() rorg.getPrimaryParent.assert_called_once_with() def test_unloadDirectory_false(self): '''test that _delObject method was not called ''' rp_dir = '/path/to/test_zp/reports/SomeReports' orgpath = '/SomeReports' rp_id = 'reportName' report_data = [(orgpath, rp_id, '/path/to/test_zp/Reports/SomeReports/reportName.rpt' )] rorg = Mock(id='Reports') rorg_parent = Mock(id='Reports') rorg._delObject = Mock() rorg.objectValues = Mock(return_value=False) rorg.getPrimaryParent = Mock(return_value=rorg_parent) setattr(rorg, rp_id, False) self.rp_load.dmd.Reports.createOrganizer = Mock(return_value=rorg) self.rp_load.reports = Mock(return_value=report_data) self.rp_load.unloadDirectory(repdir=rp_dir) rorg._delObject.assert_not_called() def test_loadDirectory_force(self): full_path = '/path/to/test_zp/Reports/SomeReports/reportName.rpt' rp_dir = '/path/to/test_zp/reports/SomeReports' orgpath = '/SomeReports' rp_id = 'reportName' report_data = [(orgpath, rp_id, full_path)] self.rp_load.options.force = True rorg = Mock() report = Mock() #set that this report is not from zenpack report.pack = Mock(return_value=False) setattr(rorg, rp_id, report) rorg._delObject = Mock() rorg.objectValues = Mock(return_value=False) self.rp_load.dmd.Reports.createOrganizer = Mock(return_value=rorg) self.rp_load.reports = Mock(return_value=report_data) self.rp_load.loadFile = create_autospec(self.rp_load.loadFile) self.rp_load.loadDirectory(rp_dir) self.rp_load.dmd.Reports.createOrganizer.assert_called_once_with(orgpath) rorg._delObject.assert_called_once_with(rp_id) self.rp_load.loadFile.assert_called_with(rorg, rp_id, full_path) def test_loadDirectory(self): '''test that _delObject method was not called and we didn't overwrite reports ''' full_path = '/path/to/test_zp/Reports/SomeReports/reportName.rpt' rp_dir = '/path/to/test_zp/reports/SomeReports' orgpath = '/SomeReports' rp_id = 'reportName' report_data = [(orgpath, rp_id, full_path)] #force option is False by default, this is for better clarity self.rp_load.options.force = False rorg = Mock() setattr(rorg, rp_id, True) rorg._delObject = Mock() rorg.objectValues = Mock(return_value=False) self.rp_load.dmd.Reports.createOrganizer = Mock(return_value=rorg) self.rp_load.reports = Mock(return_value=report_data) self.rp_load.loadFile = Mock() self.rp_load.loadDirectory(rp_dir) rorg._delObject.assert_not_called() self.rp_load.loadFile.assert_not_called() @patch( '__builtin__.file', autospec=True, spec_set=True ) def test_loadFile(self, file_mock): rp_name = 'reportName' full_rp_path = '/path/to/test_zp/Reports/SomeReports/reportName.rpt' report_txt = "some report data" #mock build in file method and its instance read method file_read = Mock() file_read.read = Mock(return_value=report_txt) file_mock.return_value = file_read root = Mock() root._setObject = Mock() rp = self.rp_load.loadFile(root, rp_name, full_rp_path) self.assertIsInstance(rp, Report) self.assertEqual(rp.id, rp_name) root._setObject.assert_called_once_with(rp_name, rp)
def build(self): self.db = None self.storage = None conn = None try: self.zodbConnect() conn = self.db.open() root = conn.root() app = root.get('Application') if app and getattr(app, self.sitename, None) is not None: print "zport portal object exists; exiting." return except self.connectionFactory.exceptions.OperationalError as e: print "zenbuild: Database does not exist." sys.exit(1) finally: if conn: conn.close() if self.db: self.db.close() self.db = None if self.storage: self.storage.close() self.storage = None # TODO: remove the port condition, in here for now because there is no SQL dump of postgresql if self.options.fromXml: self.connect() from Products.ZenModel.ZentinelPortal import manage_addZentinelPortal manage_addZentinelPortal(self.app, self.sitename) site = self.app._getOb(self.sitename) # build index_html if self.app.hasObject('index_html'): self.app._delObject('index_html') from Products.PythonScripts.PythonScript import manage_addPythonScript manage_addPythonScript(self.app, 'index_html') newIndexHtml = self.app._getOb('index_html') text = 'container.REQUEST.RESPONSE.redirect(container.zport.virtualRoot() + "/zport/dmd/")\n' newIndexHtml.ZPythonScript_edit('', text) # build standard_error_message if self.app.hasObject('standard_error_message'): self.app._delObject('standard_error_message') file = open(zenPath('Products/ZenModel/dtml/standard_error_message.dtml')) try: text = file.read() finally: file.close() import OFS.DTMLMethod OFS.DTMLMethod.addDTMLMethod(self.app, id='standard_error_message', file=text) # Convert the acl_users folder at the root to a PAS folder and update # the login form to use the Zenoss login form Security.replaceACLWithPAS(self.app, deleteBackup=True) auth0_setup(self.app) account_locker_setup(self.app) # Add groupManager to zport.acl acl = site.acl_users if not hasattr(acl, 'groupManager'): plugins.ZODBGroupManager.addZODBGroupManager(acl, 'groupManager') acl.groupManager.manage_activateInterfaces(['IGroupsPlugin',]) trans = transaction.get() trans.note("Initial ZentinelPortal load by zenbuild.py") trans.commit() print "ZentinelPortal loaded at %s" % self.sitename # build dmd from Products.ZenModel.DmdBuilder import DmdBuilder dmdBuilder = DmdBuilder(site, self.options.evthost, self.options.evtuser, self.options.evtpass, self.options.evtdb, self.options.evtport, self.options.smtphost, self.options.smtpport, self.options.pagecommand) dmdBuilder.build() transaction.commit() # Load XML Data from Products.ZenModel.XmlDataLoader import XmlDataLoader dl = XmlDataLoader(noopts=True, app=self.app) dl.loadDatabase() else: cmd = "gunzip -c %s | %s --usedb=zodb" % ( zenPath("Products/ZenModel/data/zodb.sql.gz"), zenPath("Products/ZenUtils/ZenDB.py"), ) returncode = os.system(cmd) if returncode: print >> sys.stderr, "There was a problem creating the database from the sql dump." sys.exit(1) # Relstorage may have already loaded items into the cache in the # initial connection to the database. We have to expire everything # in the cache in order to prevent errors with overlapping # transactions from the model which was just imported above. if self.options.zodb_cacheservers: self.flush_memcached(self.options.zodb_cacheservers.split()) self.connect() # Set all the attributes site = getattr(self.app, self.sitename, None) site.dmd.smtpHost = self.options.smtphost site.dmd.smtpPort = self.options.smtpport site.dmd.pageCommand = self.options.pagecommand site.dmd.uuid = None for evmgr in (site.dmd.ZenEventManager, site.dmd.ZenEventHistory): evmgr.username = self.options.evtuser evmgr.password = self.options.evtpass evmgr.database = self.options.evtdb evmgr.host = self.options.evthost evmgr.port = self.options.evtport transaction.commit() # Load reports from Products.ZenReports.ReportLoader import ReportLoader rl = ReportLoader(noopts=True, app=self.app) rl.loadDatabase()