Exemplo n.º 1
0
    def test_post5(self):

        createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
                              "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"])

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgB-0.6.0.unix.cronus')))
Exemplo n.º 2
0
 def test_parsing_good_url(self):
     # test good url
     uri = 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus?query=blah#foo'
     uriDict = PackageUtil.parseUri(uri, PackageMgr.packagePath())
     assert uriDict['uri'] == uri
     assert uriDict['scheme'] == 'http'
     assert uriDict['uripath'] == '/repo/agent-0.1.2.unix.cronus'
     assert uriDict['package'] == 'agent-0.1.2.unix.cronus'
     assert uriDict['packageNameVersion'] == 'agent-0.1.2.unix'
     assert uriDict['packageName'] == 'agent'
     assert uriDict['packageVersion'] == '0.1.2.unix'
     assert uriDict['packageVersionMajor'] == '0'
     assert uriDict['packageVersionMinor'] == '1'
     assert uriDict['packageVersionBuild'] == '2'
     assert uriDict['packagePlat'] == 'unix'
     assert uriDict['propName'] == 'agent-0.1.2.unix.cronus.prop'
     assert uriDict[
         'propUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.prop?query=blah#foo'
     assert uriDict['packagePath'] == os.path.join(
         PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus')
     assert uriDict['inProgressPackagePath'] == os.path.join(
         PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.inprogress')
     assert uriDict['propPath'] == os.path.join(
         PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.prop')
     assert uriDict[
         'torrentUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.torrent?query=blah#foo'
     assert uriDict['torrentPath'] == os.path.join(
         PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.torrent')
     assert uriDict['torrentName'] == 'agent-0.1.2.unix.cronus.torrent'
Exemplo n.º 3
0
    def test_post5(self):

        createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
                              "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"])

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgB-0.6.0.unix.cronus')))
Exemplo n.º 4
0
    def test_init(self):
        # test the init routine

        # make sure that the initial path is cleared out of IN Progress nodes
        tmpFile = os.path.join(PackageMgr.packagePath(), 'foo.inprogress')
        open(tmpFile, 'w').close()

        tmpFile2 = os.path.join(PackageMgr.packagePath(), 'foo2.inprogress')
        open(tmpFile2, 'w').close()

        stopAgentGlobals()
        startAgentGlobals()
Exemplo n.º 5
0
    def test_init(self):
        # test the init routine

        # make sure that the initial path is cleared out of IN Progress nodes
        tmpFile = os.path.join(PackageMgr.packagePath(), 'foo.inprogress')
        open(tmpFile, 'w').close()

        tmpFile2 = os.path.join(PackageMgr.packagePath(), 'foo2.inprogress')
        open(tmpFile2, 'w').close()

        stopAgentGlobals()
        startAgentGlobals()
Exemplo n.º 6
0
 def listPackages(self):
     """ list content of packages folder
     """
     result = {}
     pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ]
     result['packages'] = pkgs
     return doneResult(request, response, result = result, controller = self)
Exemplo n.º 7
0
 def listPackages(self):
     """ list content of packages folder
     """
     result = {}
     pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ]
     result['packages'] = pkgs
     return doneResult(request, response, result = result, controller = self)
Exemplo n.º 8
0
    def test_noAgeGarbageCollection(self):
        #from agent.controllers.service import ServiceController
        from agent.tests.unit.test_util import createManifest
        serviceName = 'service4'
        manifestName = 'manifestb'
        try:
            createManifest(
                self,
                packages=[
                    "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
                    "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"
                ],
                service=serviceName,
                manifest=manifestName)
        except Exception as ex:
            print 'exception thrown during mf %s' % str(ex)

        self.app.delete(url(controller='manifest',
                            action='delete',
                            service=serviceName,
                            manifest=manifestName),
                        expect_errors=True)
        age = pylons.config['packageMgr_package_age']
        pylons.config['packageMgr_package_age'] = 86400
        pkgs = PackageMgr.packagePath()
        self.assertEquals(len(os.listdir(pkgs)), 4)
        time.sleep(40)
        self.assertEquals(len(os.listdir(pkgs)), 4)
        pylons.config['packageMgr_package_age'] = age
Exemplo n.º 9
0
    def test_delete_cronus_package(self):
        package = "pkgA-1.2.0.unix"
        package_url = "http://repository.qa.ebay.com/cronus/test-data/agent/pkgA-1.2.0.unix.cronus"
        package_path = os.path.join(PackageMgr.packagePath(),
                                    package + '.cronus')
        # case 1 - package is not present
        self.assertFalse(os.path.exists(package_path))
        response = self.app.delete(url(controller='cleanup',
                                       package=package,
                                       action='deleteCronusPackage'),
                                   expect_errors=True)
        self.assertEqual(response.status_int, 200)

        # case 2 - package is present
        body = json.dumps({
            'package': package_url,
            'packageloc': package + '.cronus'
        })
        response = self.app.post(url(controller='distribution',
                                     action='startdownload',
                                     service="dist"),
                                 headers={'Content-Type': 'application/json'},
                                 params=body,
                                 expect_errors=True)

        self.assertEqual(response.status_int, 200)
        checkStatus(self, "http_download", response, 100)
        self.assertTrue(os.path.exists(package_path))
        response = self.app.delete(url(controller='cleanup',
                                       package=package,
                                       action='deleteCronusPackage'),
                                   expect_errors=True)
        self.assertEqual(response.status_int, 200)
        self.assertFalse(os.path.exists(package_path))
Exemplo n.º 10
0
    def __test_garbageCollection(self):
        from agent.tests.unit.test_util import createManifest
        serviceName = 'service3'
        manifestName = 'manifestA'
        try:
            createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
                                  "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
                                  service = serviceName, manifest = manifestName)
        except Exception as ex:
            print 'exception thrown during mf %s' % str(ex)

        self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)


        age = pylons.config['packageMgr_package_age']
        pylons.config['packageMgr_package_age'] = 10
        pkgs = PackageMgr.packagePath()
        self.assertEquals(len(os.listdir(pkgs)), 2)
        appGlobal = pylons.config['pylons.app_globals']
        appGlobal.packageMgr.start()
        time.sleep(80)
        self.assertEquals(len(os.listdir(pkgs)), 0)
        pylons.config['packageMgr_package_age'] = age
        self.assertEquals(pylons.config['packageMgr_package_age'], age)
        appGlobal.packageMgr.stop()
        time.sleep(20)
Exemplo n.º 11
0
    def test_forceDeletePackages(self):
        #import pdb;pdb.set_trace();
        from agent.tests.unit.test_util import createManifest
        if os.name == 'posix':
            serviceName = 'service4'
            manifestName = 'manifestb'
            try:
                createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
                                  "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
                                  service = serviceName, manifest = manifestName)
            except Exception as ex:
                print 'exception thrown during mf %s' % str(ex)

            self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
            age = pylons.config['packageMgr_package_age']
            pylons.config['packageMgr_package_age'] = 86400
            minage = pylons.config['packageMgr_package_min_age']
            pylons.config['packageMgr_package_min_age'] = 0.0
            #appGlobal = pylons.config['pylons.app_globals']
            #appGlobal.packageMgr.start()
            disk_threshold = pylons.config['health_disk_usage_percent_threshold']
            pylons.config['health_disk_usage_percent_threshold'] = 1
            pylons.config['health_disk_usage_gc_threshold'] = 1

            pkgs = PackageMgr.packagePath()
            self.assertEquals(len(os.listdir(pkgs)), 2)
            PackageMgr.forceCleanUpDownloadedPkgs()
            self.assertEquals(len(os.listdir(pkgs)), 0)
            pylons.config['packageMgr_package_age'] = age
            pylons.config['packageMgr_package_min_age'] = minage
            pylons.config['health_disk_usage_percent_threshold'] = disk_threshold
Exemplo n.º 12
0
 def test_parsing_bad_extension(self):
     # test bad extension
     try:
         PackageUtil.parseUri('http://foo.com/repo/agent-0.1.1a.cronus2', PackageMgr.packagePath())
         assert False, 'bad extension'
     except AgentException:
         pass
Exemplo n.º 13
0
    def test_package_min_age_during_forceDelete(self):
        #import pdb;pdb.set_trace();
        from agent.tests.unit.test_util import createManifest
        serviceName = 'service5'
        manifestName = 'manifestb'
        try:
            createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
                                  "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
                                  service = serviceName, manifest = manifestName)
        except Exception as ex:
            print 'exception thrown during mf %s' % str(ex)

        self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
        age = pylons.config['packageMgr_package_age']
        pylons.config['packageMgr_package_age'] = 86400
        minage = pylons.config['packageMgr_package_min_age']
        pylons.config['packageMgr_package_min_age'] = 86400

        disk_threshold = pylons.config['health_disk_usage_percent_threshold']
        pylons.config['health_disk_usage_percent_threshold'] = 1

        pkgs = PackageMgr.packagePath()
        self.assertEquals(len(os.listdir(pkgs)), 2)
        time.sleep(40)
        self.assertEquals(len(os.listdir(pkgs)), 2)
        pylons.config['packageMgr_package_age'] = age
        pylons.config['packageMgr_package_min_age'] = minage
        pylons.config['health_disk_usage_percent_threshold'] = disk_threshold
Exemplo n.º 14
0
 def test_parsing_bad_name(self):
     # test bad name
     try:
         PackageUtil.parseUri('http://foo.com/repo/a?gent-0.1.1.unix.cronus', PackageMgr.packagePath())
         assert False, 'bad name'
     except AgentException:
         pass
Exemplo n.º 15
0
    def setUp(self):
        commonSetup()

        self.testPkgUri = 'http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.1.0.unix.cronus'
        self.localPkgName = os.path.join(PackageMgr.packagePath(), 'perlserver-1.1.0.unix.cronus')
        self.threadMgr = pylons.config['pylons.app_globals'].threadMgr
        appGlobals = pylons.config['pylons.app_globals']
        appGlobals.packageMgr.stop()
Exemplo n.º 16
0
 def test_parsing_bad_extension(self):
     # test bad extension
     try:
         PackageUtil.parseUri('http://foo.com/repo/agent-0.1.1a.cronus2',
                              PackageMgr.packagePath())
         assert False, 'bad extension'
     except AgentException:
         pass
Exemplo n.º 17
0
 def getPkgUploadPage(self):
     """ show existing packages in packages folder """
     pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ]
     c.content = pkgs
     
     menuTab = "MENU_TAB_PKGS"
     c.menuTab = menuTab        
     return render('/derived/pkgUpload.html')
Exemplo n.º 18
0
    def test_post2(self):
        # successful post
        createManifest(self)

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
        self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
                                           'cronus', 'scripts', 'activate')))
        self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
Exemplo n.º 19
0
 def test_parsing_bad_name(self):
     # test bad name
     try:
         PackageUtil.parseUri(
             'http://foo.com/repo/a?gent-0.1.1.unix.cronus',
             PackageMgr.packagePath())
         assert False, 'bad name'
     except AgentException:
         pass
Exemplo n.º 20
0
    def setUp(self):
        commonSetup()

        self.testPkgUri = 'http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.1.0.unix.cronus'
        self.localPkgName = os.path.join(PackageMgr.packagePath(),
                                         'perlserver-1.1.0.unix.cronus')
        self.threadMgr = pylons.config['pylons.app_globals'].threadMgr
        appGlobals = pylons.config['pylons.app_globals']
        appGlobals.packageMgr.stop()
Exemplo n.º 21
0
    def test_inprogress_pkg_download(self):
        service = 'foo'
        try:
            path = ServiceController.manifestPath(service)
            os.makedirs(path)
            path = ServiceController.installedPkgPath(service)
            os.makedirs(path)
            path = ServiceController.downloadedPkgPath(service)
            os.makedirs(path)
            inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress')
            inprogressFile = open(inprogressPath, 'w')
            inprogressFile.write('somegarbage')
            inprogressFile.close()
        except Exception as excep:
            LOG.warning('got an OS Exception - %s' % str(excep))

        createManifest(self)
        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
        self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
Exemplo n.º 22
0
    def test_parsing_garbage(self):
        # test the url parsing

        try:
            PackageUtil.parseUri(
                'asdfawert892375[jrgjhnjbuy43w5897t^!R$#W_R(@$^(_#$(',
                PackageMgr.packagePath())
            assert False, 'parse garbage'
        except AgentException:
            pass
Exemplo n.º 23
0
    def test_inprogress_pkg_download(self):
        service = 'foo'
        try:
            path = ServiceController.manifestPath(service)
            os.makedirs(path)
            path = ServiceController.installedPkgPath(service)
            os.makedirs(path)
            path = ServiceController.downloadedPkgPath(service)
            os.makedirs(path)
            inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress')
            inprogressFile = open(inprogressPath, 'w')
            inprogressFile.write('somegarbage')
            inprogressFile.close()
        except Exception as excep:
            LOG.warning('got an OS Exception - %s' % str(excep))

        createManifest(self)
        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
        self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
Exemplo n.º 24
0
 def test_parsing_good_url(self):
     # test good url
     uri = 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus?query=blah#foo'
     uriDict = PackageUtil.parseUri(uri, PackageMgr.packagePath())
     assert uriDict['uri'] == uri
     assert uriDict['scheme'] == 'http'
     assert uriDict['uripath'] == '/repo/agent-0.1.2.unix.cronus'
     assert uriDict['package'] == 'agent-0.1.2.unix.cronus'
     assert uriDict['packageNameVersion'] == 'agent-0.1.2.unix'
     assert uriDict['packageName'] == 'agent'
     assert uriDict['packageVersion'] == '0.1.2.unix'
     assert uriDict['packageVersionMajor'] == '0'
     assert uriDict['packageVersionMinor'] == '1'
     assert uriDict['packageVersionBuild'] == '2'
     assert uriDict['packagePlat'] == 'unix'
     assert uriDict['propName'] == 'agent-0.1.2.unix.cronus.prop'
     assert uriDict['propUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.prop?query=blah#foo'
     assert uriDict['packagePath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus')
     assert uriDict['inProgressPackagePath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.inprogress')
     assert uriDict['propPath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.prop')
Exemplo n.º 25
0
 def _classfiy_packages(self, packages):
     """ verify that all packages that need to be downloaded exist """
     non_existent_packages = []
     filtered_packages = []
     packagePath = PackageMgr.packagePath()
     for one_package in packages:
         path = os.path.join(packagePath, one_package + '.cronus')
         if not os.path.exists(path):
             non_existent_packages.append(one_package)
         else:
             filtered_packages.append(one_package)
     return non_existent_packages, filtered_packages
Exemplo n.º 26
0
 def _classfiy_packages(self, packages):
     """ verify that all packages that need to be downloaded exist """
     non_existent_packages = []
     filtered_packages = []
     packagePath = PackageMgr.packagePath()
     for one_package in packages:
         path = os.path.join(packagePath, one_package + '.cronus')
         if not os.path.exists(path):
             non_existent_packages.append(one_package)
         else:
             filtered_packages.append(one_package)
     return non_existent_packages, filtered_packages
Exemplo n.º 27
0
    def test_post3_pkg_already_installed(self):
        createManifest(self)

        # now lets remove the manifest path
        path = os.path.join(manifestutil.manifestPath('foo', 'bar'))
        shutil.rmtree(path)

        # now create the manifest again
        createManifest(self)

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
        self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
                                           'cronus', 'scripts', 'activate')))
        self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
Exemplo n.º 28
0
    def deletePackage(self, package):
        """ secret !!hack!! API to delete one cronus package in the packages folder. """
        try:
            LOG.info('secret delete called for package %s' % package)
            packagesPath = PackageMgr.packagePath()
            thisPkgPath = os.path.join(packagesPath, package + '.cronus')
            thisPropPath = os.path.join(packagesPath, package + '.cronus.prop')
            thisInprogressPath = PackageUtil.inProgressPath(thisPkgPath)

            PackageUtil.cleanUpPackage(thisInprogressPath, thisPkgPath, thisPropPath)
            return doneResult(request, response, controller=self)
        except Exception as excp:
            return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
                               errorMsg = 'Unknown error when deleting package %s,  %s - %s' %
                               (package, str(excp), traceback.format_exc(2)),
                               controller = self)
Exemplo n.º 29
0
    def test_post3_already_installed_manifest(self):
        createManifest(self)

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
        self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
                                           'cronus', 'scripts', 'activate')))
        self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))


        body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']})
        response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"),
                                 headers = {'Content-Type' : 'application/json'},
                                 params = body, expect_errors = True)
        self.assertEquals(201, response.status_int, 'Manifest Post assert')
        body = json.loads(response.body)
        assert response.status_int == 201, 'Manifest Post assert'
Exemplo n.º 30
0
    def deletePackage(self, package):
        """ secret !!hack!! API to delete one cronus package in the packages folder. """
        try:
            LOG.info('secret delete called for package %s' % package)
            packagesPath = PackageMgr.packagePath()
            thisPkgPath = os.path.join(packagesPath, package + '.cronus')
            thisPropPath = os.path.join(packagesPath, package + '.cronus.prop')
            thisInprogressPath = PackageUtil.inProgressPath(thisPkgPath)

            PackageUtil.cleanUpPackage(thisInprogressPath, thisPkgPath, thisPropPath)
            return doneResult(request, response, controller=self)
        except Exception as excp:
            return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
                               errorMsg = 'Unknown error when deleting package %s,  %s - %s' %
                               (package, str(excp), traceback.format_exc(2)),
                               controller = self)
Exemplo n.º 31
0
    def test_post3_already_installed_manifest(self):
        createManifest(self)

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
        self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
                                           'cronus', 'scripts', 'activate')))
        self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))


        body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']})
        response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"),
                                 headers = {'Content-Type' : 'application/json'},
                                 params = body, expect_errors = True)
        self.assertEquals(201, response.status_int, 'Manifest Post assert')
        body = json.loads(response.body)
        assert response.status_int == 201, 'Manifest Post assert'
Exemplo n.º 32
0
    def test_package_reuse(self):
        createManifest(self)
        body = json.dumps({'package' : ['/packages/perlserver']})
        body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"),
                                 headers = {'Content-Type' : 'application/json'},
                                 params = body, expect_errors = True)

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
        self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
                                           'cronus', 'scripts', 'activate')))
        self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))

        for _ in range(10):
            if islink(os.path.join(manifestutil.manifestPath('foo', 'baz'), 'perlserver')):
                break
            time.sleep(1)

        self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'baz'), 'perlserver')))
Exemplo n.º 33
0
    def test_package_reuse(self):
        createManifest(self)
        body = json.dumps({'package' : ['/packages/perlserver']})
        body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"),
                                 headers = {'Content-Type' : 'application/json'},
                                 params = body, expect_errors = True)

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
        self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
                                           'cronus', 'scripts', 'activate')))
        self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))

        for _ in range(10):
            if islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')):
                break
            time.sleep(1)

        self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')))
Exemplo n.º 34
0
    def forceCleanUpDownloadedPkgs():
        ''' tries to cleanup packages from oldest date to latest ignoring packages downloaded in the last hour'''
        import operator
        import glob
        import time
        from agent.lib.packagemgr import PackageMgr

        LOG.info('Force garbage collecting downloaded packages')
        packageMount = pylons.config['agent_root']
        lower_age_bound = float(pylons.config['packageMgr_package_min_age'])
        cronus_pkgs = glob.glob(os.path.join(PackageMgr.packagePath(), '*.cronus'))
        file_list = []
        for filename in cronus_pkgs:
            try:
                date_file_tuple = os.path.getatime(filename), os.path.getsize(filename), filename
                file_list.append(date_file_tuple)
            except OSError, osErr:
                LOG.error('Unable to get access time %s - %s' % (filename, osErr))
Exemplo n.º 35
0
    def test_post3_pkg_already_installed(self):
        createManifest(self)

        # now lets remove the manifest path
        path = os.path.join(ManifestController.manifestPath('foo', 'bar'))
        if os.name == 'nt':
            cmd = 'rm -r %s' % path
            LOG.debug("running command %s" % cmd)
            os.system(cmd)
        else:
            shutil.rmtree(path)

        # now create the manifest again
        createManifest(self)

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
        self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
                                           'cronus', 'scripts', 'activate')))
        self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
Exemplo n.º 36
0
    def test_forceDeletePackages(self):
        #import pdb;pdb.set_trace();
        from agent.tests.unit.test_util import createManifest
        if os.name == 'posix':
            serviceName = 'service4'
            manifestName = 'manifestb'
            try:
                createManifest(
                    self,
                    packages=[
                        "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
                        "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"
                    ],
                    service=serviceName,
                    manifest=manifestName)
            except Exception as ex:
                print 'exception thrown during mf %s' % str(ex)

            self.app.delete(url(controller='manifest',
                                action='delete',
                                service=serviceName,
                                manifest=manifestName),
                            expect_errors=True)
            age = pylons.config['packageMgr_package_age']
            pylons.config['packageMgr_package_age'] = 86400
            minage = pylons.config['packageMgr_package_min_age']
            pylons.config['packageMgr_package_min_age'] = 0.0
            #appGlobal = pylons.config['pylons.app_globals']
            #appGlobal.packageMgr.start()
            disk_threshold = pylons.config[
                'health_disk_usage_percent_threshold']
            pylons.config['health_disk_usage_percent_threshold'] = 1
            pylons.config['health_disk_usage_gc_threshold'] = 1

            pkgs = PackageMgr.packagePath()
            self.assertEquals(len(os.listdir(pkgs)), 4)
            PackageUtil.forceCleanUpDownloadedPkgs()
            self.assertEquals(len(os.listdir(pkgs)), 0)
            pylons.config['packageMgr_package_age'] = age
            pylons.config['packageMgr_package_min_age'] = minage
            pylons.config[
                'health_disk_usage_percent_threshold'] = disk_threshold
Exemplo n.º 37
0
    def test_same_pkg_download_parallel(self):
        packages = ['http://www.stackscaling.com/agentrepo/pkgA-1.2.0.unix.cronus']
        manifest1 = 'bar'
        manifest2 = 'blah'
        service1 = 'foo'
        service2 = 'lah'
        try:
            for pkg in packages:
                mockDownloadPkg(pkg)
            
            path = ServiceController.manifestPath(service1)
            os.makedirs(path)
            path = ServiceController.installedPkgPath(service1)
            os.makedirs(path)
            path = ServiceController.downloadedPkgPath(service1)
            os.makedirs(path)
            path = ServiceController.manifestPath(service2)
            os.makedirs(path)
            path = ServiceController.installedPkgPath(service2)
            os.makedirs(path)
            path = ServiceController.downloadedPkgPath(service2)
            os.makedirs(path)
        except Exception as excep:
            LOG.warning('got an OS Exception - %s' % str(excep))

        body = json.dumps({'package' : packages})
        response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1),
                                 headers = {'Content-Type' : 'application/json'},
                                 params = body)
        assert response1.status_int == 200, 'Manifest Post assert'

        response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2),
                                 headers = {'Content-Type' : 'application/json'},
                                 params = body)
        assert response2.status_int == 200, 'Manifest Post assert'

        checkStatus(self, 'create manifest bar', response1, timeout = 25)
        checkStatus(self, 'create manifest baz', response2, timeout = 25)

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
        self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'pkgA')))
        self.assertTrue(islink(os.path.join(manifestutil.manifestPath('lah', 'blah'), 'pkgA')))
Exemplo n.º 38
0
    def test_noAgeGarbageCollection(self):
        #from agent.controllers.service import ServiceController
        from agent.tests.unit.test_util import createManifest
        serviceName = 'service4'
        manifestName = 'manifestb'
        try:
            createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
                                  "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
                                  service = serviceName, manifest = manifestName)
        except Exception as ex:
            print 'exception thrown during mf %s' % str(ex)

        self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
        age = pylons.config['packageMgr_package_age']
        pylons.config['packageMgr_package_age'] = 86400
        pkgs = PackageMgr.packagePath()
        self.assertEquals(len(os.listdir(pkgs)), 2)
        time.sleep(40)
        self.assertEquals(len(os.listdir(pkgs)), 2)
        pylons.config['packageMgr_package_age'] = age
Exemplo n.º 39
0
    def test_getPackagePaths(self):
        serviceName = 'service1'
        manifestName = 'manifestA'
        createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
                              "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
                              service = serviceName, manifest = manifestName)

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))

        packagePaths = manifestutil.packagePaths(serviceName, manifestName);
        print ">>>>>>>>>>>>" + str(packagePaths)
        self.assertTrue(len(packagePaths) >= 2)
        bitmap = 0
        for path in packagePaths:
            if path.find("pkgA") >= 0:
                self.assertTrue(os.path.isdir(path))
                bitmap |= 1
            elif path.find("pkgB") >= 0:
                self.assertTrue(os.path.isdir(path))
                bitmap |= 2
        self.assertEquals(3, bitmap)
Exemplo n.º 40
0
    def test_getPackagePaths(self):
        serviceName = 'service1'
        manifestName = 'manifestA'
        createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
                              "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
                              service = serviceName, manifest = manifestName)

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))

        packagePaths = ManifestController.getPackagePaths(serviceName, manifestName);
        print ">>>>>>>>>>>>" + str(packagePaths)
        self.assertTrue(len(packagePaths) >= 2)
        bitmap = 0
        for path in packagePaths:
            if path.find("pkgA") >= 0:
                self.assertTrue(os.path.isdir(path))
                bitmap |= 1
            elif path.find("pkgB") >= 0:
                self.assertTrue(os.path.isdir(path))
                bitmap |= 2
        self.assertEquals(3, bitmap)
Exemplo n.º 41
0
    def forceCleanUpDownloadedPkgs():
        ''' tries to cleanup packages from oldest date to latest ignoring packages downloaded in the last hour'''
        import operator
        import glob
        import time
        from agent.lib.packagemgr import PackageMgr

        LOG.info('Force garbage collecting downloaded packages')
        packageMount = pylons.config['agent_root']
        lower_age_bound = float(pylons.config['packageMgr_package_min_age'])
        cronus_pkgs = glob.glob(
            os.path.join(PackageMgr.packagePath(), '*.cronus'))
        file_list = []
        for filename in cronus_pkgs:
            try:
                date_file_tuple = os.path.getatime(filename), os.path.getsize(
                    filename), filename
                file_list.append(date_file_tuple)
            except OSError, osErr:
                LOG.error('Unable to get access time %s - %s' %
                          (filename, osErr))
Exemplo n.º 42
0
    def test_delete_cronus_package(self):
        package = "pkgA-1.2.0.unix"
        package_url = "http://repository.qa.ebay.com/cronus/test-data/agent/pkgA-1.2.0.unix.cronus"
        package_path = os.path.join(PackageMgr.packagePath(), package + '.cronus')
        # case 1 - package is not present
        self.assertFalse(os.path.exists(package_path))
        response = self.app.delete(url(controller='cleanup', package=package, action='deleteCronusPackage'), expect_errors = True)
        self.assertEqual(response.status_int, 200)

        # case 2 - package is present
        body = json.dumps({'package': package_url, 'packageloc' : package + '.cronus'})
        response = self.app.post(url(controller = 'distribution', action = 'startdownload', service = "dist"),
                                 headers = {'Content-Type' : 'application/json'},
                                 params = body, expect_errors = True)

        self.assertEqual(response.status_int, 200)
        checkStatus(self, "http_download", response, 100)
        self.assertTrue(os.path.exists(package_path))
        response = self.app.delete(url(controller='cleanup', package=package, action='deleteCronusPackage'), expect_errors = True)
        self.assertEqual(response.status_int, 200)
        self.assertFalse(os.path.exists(package_path))
Exemplo n.º 43
0
    def test_same_pkg_download_parallel(self):
        packages = ['http://repository.qa.ebay.com/cronus/test-data/agent/perlserver-1.0.0.unix.cronus']
        manifest1 = 'bar'
        manifest2 = 'blah'
        service1 = 'foo'
        service2 = 'lah'
        try:
            path = ServiceController.manifestPath(service1)
            os.makedirs(path)
            path = ServiceController.installedPkgPath(service1)
            os.makedirs(path)
            path = ServiceController.downloadedPkgPath(service1)
            os.makedirs(path)
            path = ServiceController.manifestPath(service2)
            os.makedirs(path)
            path = ServiceController.installedPkgPath(service2)
            os.makedirs(path)
            path = ServiceController.downloadedPkgPath(service2)
            os.makedirs(path)
        except Exception as excep:
            LOG.warning('got an OS Exception - %s' % str(excep))

        body = json.dumps({'package' : packages})
        response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1),
                                 headers = {'Content-Type' : 'application/json'},
                                 params = body)
        assert response1.status_int == 200, 'Manifest Post assert'

        response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2),
                                 headers = {'Content-Type' : 'application/json'},
                                 params = body)
        assert response2.status_int == 200, 'Manifest Post assert'

        checkStatus(self, 'create manifest bar', response1, timeout = 25)
        checkStatus(self, 'create manifest baz', response2, timeout = 25)

        self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
        self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
        self.assertTrue(islink(os.path.join(ManifestController.manifestPath('lah', 'blah'), 'perlserver')))
Exemplo n.º 44
0
    def test_cleanupPackages(self):
        stopAgentGlobals()

        # ok now let's create the directory structure of a bad service and packages
        packagePath = PackageMgr.packagePath()
        inProgPackage2 = os.path.join(packagePath, 'bar.inprogress')
        inProgPackage1 = os.path.join(packagePath, 'foo.inprogress')
        goodPackage1 = os.path.join(packagePath, 'baz')
        goodPackage2 = os.path.join(packagePath, 'baz')

        open(inProgPackage1, 'w').close()
        open(inProgPackage2, 'w').close()
        open(goodPackage1, 'w').close()
        open(goodPackage2, 'w').close()

        startAgentGlobals()
        # we remove logic of inprogress file cleanup from packagemgr
#        assert not os.path.exists(inProgPackage1)
#        assert not os.path.exists(inProgPackage2)

        assert os.path.exists(goodPackage1)
        assert os.path.exists(goodPackage2)
Exemplo n.º 45
0
    def test_cleanupPackages(self):
        stopAgentGlobals()

        # ok now let's create the directory structure of a bad service and packages
        packagePath = PackageMgr.packagePath()
        inProgPackage2 = os.path.join(packagePath, 'bar.inprogress')
        inProgPackage1 = os.path.join(packagePath, 'foo.inprogress')
        goodPackage1 = os.path.join(packagePath, 'baz')
        goodPackage2 = os.path.join(packagePath, 'baz')

        open(inProgPackage1, 'w').close()
        open(inProgPackage2, 'w').close()
        open(goodPackage1, 'w').close()
        open(goodPackage2, 'w').close()

        startAgentGlobals()
        # we remove logic of inprogress file cleanup from packagemgr
        #        assert not os.path.exists(inProgPackage1)
        #        assert not os.path.exists(inProgPackage2)

        assert os.path.exists(goodPackage1)
        assert os.path.exists(goodPackage2)
Exemplo n.º 46
0
    def test_package_min_age_during_forceDelete(self):
        #import pdb;pdb.set_trace();
        from agent.tests.unit.test_util import createManifest
        serviceName = 'service5'
        manifestName = 'manifestb'
        try:
            createManifest(
                self,
                packages=[
                    "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
                    "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"
                ],
                service=serviceName,
                manifest=manifestName)
        except Exception as ex:
            print 'exception thrown during mf %s' % str(ex)

        self.app.delete(url(controller='manifest',
                            action='delete',
                            service=serviceName,
                            manifest=manifestName),
                        expect_errors=True)
        age = pylons.config['packageMgr_package_age']
        pylons.config['packageMgr_package_age'] = 86400
        minage = pylons.config['packageMgr_package_min_age']
        pylons.config['packageMgr_package_min_age'] = 86400

        disk_threshold = pylons.config['health_disk_usage_percent_threshold']
        pylons.config['health_disk_usage_percent_threshold'] = 1

        pkgs = PackageMgr.packagePath()
        self.assertEquals(len(os.listdir(pkgs)), 4)
        time.sleep(40)
        self.assertEquals(len(os.listdir(pkgs)), 4)
        pylons.config['packageMgr_package_age'] = age
        pylons.config['packageMgr_package_min_age'] = minage
        pylons.config['health_disk_usage_percent_threshold'] = disk_threshold
Exemplo n.º 47
0
    def parseUri(packageUri, path=None, packageloc=None):
        """ static method to parse the uri
        @throws - AgentException
        """

        if (path == None):
            from agent.lib.packagemgr import PackageMgr
            path = PackageMgr.packagePath()

        uri = urlparse.urlparse(packageUri)

        if (uri.scheme != 'http'):
            raise AgentException(
                Errors.PACKAGE_SCHEME_ERROR,
                'uri (%s) scheme(%s) not supported' % (packageUri, uri.scheme))

        if (uri.path == ''):
            raise AgentException(Errors.PACKAGE_PATH_ERROR,
                                 'uri (%s) path is empty' % (packageUri))

        # now parse the path.  get the name and then verify that it matches the convention
        if packageloc is not None:
            # check if torrent download
            if uri.path.endswith('.torrent'):
                if not packageloc.endswith('.torrent'):
                    packageloc = packageloc + '.torrent'
            packName = packageloc
        else:
            packName = uri.path.rpartition('/')[2]

        match = PackageUtil.nameRe.match(packName)
        if (match == None or match.group(0) != packName):
            raise AgentException(
                Errors.PACKAGE_PATH_ERROR,
                'cannot find package name in path %s' % (uri.path))

        # ok now we can fill out the dictionary
        uriDict = {}
        uriDict['uri'] = packageUri
        uriDict['scheme'] = uri.scheme
        uriDict['uripath'] = uri.path
        uriDict['package'] = match.group(1)
        uriDict['packageNameVersion'] = match.group(2)
        uriDict['inProgressPackage'] = PackageUtil.inProgressPath(
            uriDict['package'])
        uriDict['packageName'] = match.group(3)
        uriDict['propName'] = uriDict['package'] + '.prop'
        uriDict['packageVersion'] = match.group(4)
        uriDict['packageVersionMajor'] = match.group(5)
        uriDict['packageVersionMinor'] = match.group(6)
        uriDict['packageVersionBuild'] = match.group(7)
        uriDict['packagePlat'] = match.group(8)

        # path specific attributes
        uriDict['packagePath'] = os.path.join(path, uriDict['package'])
        uriDict['inProgressPackagePath'] = os.path.join(
            path, uriDict['inProgressPackage'])
        uriDict['propPath'] = os.path.join(path, uriDict['propName'])

        # check if bittorrent transfer and define the necessary keys
        if match.group(9) is not None and match.group(9) == '.torrent':
            uriDict['scheme'] = 'bittorrent'
        uriDict['torrentName'] = uriDict['package'] + '.torrent'
        uriDict['torrentPath'] = os.path.join(path, uriDict['torrentName'])

        # calculate prop url
        # append path with .prop - mons: leaving the below code in place in case we support other protocols in future
        uriScheme = uri.scheme
        if (uriScheme !=
                "http"):  #only use http to download .prop and .torrent files
            uriScheme = "http"

        if uri.path.endswith('.torrent'):
            uripath = uri.path.strip('.torrent')
        else:
            uripath = uri.path

        propParseResult = urlparse.ParseResult(uriScheme, uri.netloc,
                                               uripath + '.prop', uri.params,
                                               uri.query, uri.fragment)
        propUri = urlparse.urlunparse(propParseResult)
        uriDict['propUri'] = propUri

        # calculate the torrent url
        # append path with .torrent
        torrentParseResult = urlparse.ParseResult(uriScheme, uri.netloc,
                                                  uripath + '.torrent',
                                                  uri.params, uri.query,
                                                  uri.fragment)
        torrentUri = urlparse.urlunparse(torrentParseResult)
        uriDict['torrentUri'] = torrentUri

        return uriDict
Exemplo n.º 48
0
    def testDelete(self):
        def deleteTestDir(service):
            LOG.debug('************ service = %s' % service)
            response = self.app.delete(url(controller='service', service=service, action='delete'), expect_errors = True)
            # make sure the responses are correct
            LOG.debug('status = ' + str(response.status_int))
#            assert response.status_int == 500, "HTTP response != 500"

        def makeTestDir(path):
            os.makedirs(path)
            os.makedirs(os.path.join(path, 'manifests'))
            os.makedirs(os.path.join(path, 'installed-packages'))

        def createManifests(mf_path):
            os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
            os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
            latest = os.path.join(mf_path, 'm3.0')
            os.makedirs(os.path.join(latest, 'dummy_dir3.0'))

            utils.symlink(latest, os.path.join(mf_path, 'active'))

            return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')

        def makePackageContent(path, pkgPath, pkgPropPath):
            pkgFile = file(pkgPath, 'w')
            for index in range(10):
                pkgFile.write(('%s%s') % (index, index))
            pkgFile.close()

            pkgFile = file(pkgPropPath, 'w')
            for index in range(10):
                pkgFile.write(('%s%s') % (index, index))
            pkgFile.close()
            uname = pylons.config['agent_user_account']
            TestCleanupController.rchown(path, uname)

        def createTestThread(serviceName):
            appGlobal = config['pylons.app_globals']
            testTh = WaitThread(appGlobal.threadMgr, ServiceController.serviceCat(serviceName))
            testTh.start()
            return testTh

        def startTestProcess():
            cmd = utils.sudoCmd(["sleep", "5"], pylons.config['app_user_account'])
            return Popen(cmd)

        path1 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'foo')
        path2 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'bar')
        path3 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'agent')

        deleteTestDir('foo')
        deleteTestDir('bar')
        deleteTestDir('agent')

        # make dirs
        makeTestDir(path1)
        makeTestDir(path2)
        makeTestDir(path3)
        all_mf, active_mf = createManifests(ServiceController.manifestPath('agent'))

        uname = pylons.config['agent_user_account']
        TestCleanupController.rchown(ServiceController.serviceRootPath(), uname)

        pkgDir = PackageMgr.packagePath()
        pkgPath = os.path.join(pkgDir, "foo.cronus")
        pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop")
        makePackageContent(pkgDir, pkgPath, pkgPropPath)

        # create threads
        testThFoo = createTestThread('foo')
        testThBar = createTestThread('bar')
        testThAgent = createTestThread('agent')

        # start process
        process = startTestProcess()

        # start testing
        LOG.debug('************ start cleanup')
        response = self.app.post(url(controller='cleanup', action='post'))
        LOG.debug ('Delete response body = ' + response.body)
        body = json.loads(response.body)

        tm = time.time()
        while (tm + 10 > time.time()):
            response = self.app.get(body['status'], expect_errors = True)
            LOG.debug ('Status response body = ' + response.body)
            body = json.loads(response.body)
            print body
            if (body['progress'] == 100):
                break
            time.sleep(0.1)

        # make sure the responses are correct
        LOG.debug('status = ' + str(response.status_int))
        assert response.status_int == 200, "HTTP response != 200"

        time.sleep(0.1)
        assert not os.path.exists(path1), 'service foo does exist or is not a directory'
        assert not os.path.exists(path2), 'service bar does exist or is not a directory'
        assert os.path.exists(path3), 'service agent does NOT exist or is not a directory'
        assert not testThFoo.isAlive(), 'thread Foo is still alive'
        assert not testThBar.isAlive(), 'thread Bar is still alive'
        assert not testThAgent.isAlive(), 'thread Agent is still alive'
        assert not os.path.exists(pkgPath), 'package foo exists'
        assert not os.path.exists(pkgPropPath), 'package prop foo exists'
        assert os.path.exists(pkgDir), 'package directory does not exist'
        # ensure agent cleanup is proper
        active_mf_path = manifestutil.manifestPath('agent', active_mf)
        active_link = os.path.join(ServiceController.manifestPath('agent'), 'active')
        all_mf.remove(active_mf)
        actual_active_mf_path = utils.readlink(active_link)

        self.assertTrue(os.path.exists(active_mf_path), 'active agent manifest got deleted but shouldn\t have')
        self.assertTrue(os.path.exists(active_link), 'agent active link missing')
        self.assertEqual(active_mf_path, actual_active_mf_path, 'agent active link pointing to some wrong manifest; link broken?')
        for mf in all_mf:
            agnt_mf_path = manifestutil.manifestPath('agent', mf)
            self.assertFalse(os.path.exists(agnt_mf_path), 'non active agent mf %s should have been deleted' % mf)
Exemplo n.º 49
0
    def test_parsing_garbage(self):
        # test the url parsing

        try:
            PackageUtil.parseUri('asdfawert892375[jrgjhnjbuy43w5897t^!R$#W_R(@$^(_#$(', PackageMgr.packagePath())
            assert False, 'parse garbage'
        except AgentException:
            pass
Exemplo n.º 50
0
    def parseUri(packageUri, path = None, packageloc = None):
        """ static method to parse the uri
        @throws - AgentException
        """

        if (path == None):
            from agent.lib.packagemgr import PackageMgr
            path = PackageMgr.packagePath()

        uri = urlparse.urlparse(packageUri)

        if (uri.scheme != 'http'):
            raise AgentException(Errors.PACKAGE_SCHEME_ERROR,
                                 'uri (%s) scheme(%s) not supported' % (packageUri, uri.scheme))

        if (uri.path == ''):
            raise AgentException(Errors.PACKAGE_PATH_ERROR,
                                 'uri (%s) path is empty' % (packageUri))

        # now parse the path.  get the name and then verify that it matches the convention
        if packageloc is not None:
            # check if torrent download
            if uri.path.endswith('.torrent'):
                if not packageloc.endswith('.torrent'):
                    packageloc = packageloc + '.torrent'
            packName = packageloc
        else:
            packName = uri.path.rpartition('/')[2]

        match = PackageUtil.nameRe.match(packName)
        if (match == None or match.group(0) != packName):
            raise AgentException(Errors.PACKAGE_PATH_ERROR,
                                 'cannot find package name in path %s' % (uri.path))

        # ok now we can fill out the dictionary
        uriDict = {}
        uriDict['uri'] = packageUri
        uriDict['scheme'] = uri.scheme
        uriDict['uripath'] = uri.path
        uriDict['package'] = match.group(1)
        uriDict['packageNameVersion'] = match.group(2)
        uriDict['inProgressPackage'] = PackageUtil.inProgressPath(uriDict['package'])
        uriDict['packageName'] = match.group(3)
        uriDict['propName'] = uriDict['package'] + '.prop'
        uriDict['packageVersion'] = match.group(4)
        uriDict['packageVersionMajor'] = match.group(5)
        uriDict['packageVersionMinor'] = match.group(6)
        uriDict['packageVersionBuild'] = match.group(7)
        uriDict['packagePlat'] = match.group(8)

        # path specific attributes
        uriDict['packagePath'] = os.path.join(path, uriDict['package'])
        uriDict['inProgressPackagePath'] = os.path.join(path, uriDict['inProgressPackage'])
        uriDict['propPath'] = os.path.join(path, uriDict['propName'])

        # check if bittorrent transfer and define the necessary keys
        if match.group(9) is not None and match.group(9) == '.torrent':
            uriDict['scheme'] = 'bittorrent'
        uriDict['torrentName'] = uriDict['package'] + '.torrent'
        uriDict['torrentPath'] = os.path.join(path, uriDict['torrentName'])

        # calculate prop url
        # append path with .prop - mons: leaving the below code in place in case we support other protocols in future
        uriScheme = uri.scheme
        if (uriScheme != "http"): #only use http to download .prop and .torrent files
            uriScheme = "http"

        if uri.path.endswith('.torrent'):
            uripath = uri.path.strip('.torrent')
        else:
            uripath = uri.path

        propParseResult = urlparse.ParseResult(uriScheme, uri.netloc, uripath + '.prop',
                                               uri.params, uri.query, uri.fragment)
        propUri = urlparse.urlunparse(propParseResult)
        uriDict['propUri'] = propUri

        # calculate the torrent url
        # append path with .torrent
        torrentParseResult = urlparse.ParseResult(uriScheme, uri.netloc, uripath + '.torrent',
                                               uri.params, uri.query, uri.fragment)
        torrentUri = urlparse.urlunparse(torrentParseResult)
        uriDict['torrentUri'] = torrentUri

        return uriDict
Exemplo n.º 51
0
 def __deletePackages(self):
     """ delete all packages from pakcage directory """
     package_path = PackageMgr.packagePath()
     ServiceDelete.deleteFolderContents(package_path, onlyChildren=True)
Exemplo n.º 52
0
    def testDelete(self):
        def deleteTestDir(service):
            LOG.debug('************ service = %s' % service)
            response = self.app.delete(url(controller='service',
                                           service=service,
                                           action='delete'),
                                       expect_errors=True)
            # make sure the responses are correct
            LOG.debug('status = ' + str(response.status_int))
#            assert response.status_int == 500, "HTTP response != 500"

        def makeTestDir(path):
            os.makedirs(path)
            os.makedirs(os.path.join(path, 'manifests'))
            os.makedirs(os.path.join(path, 'installed-packages'))

        def createManifests(mf_path):
            os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
            os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
            latest = os.path.join(mf_path, 'm3.0')
            os.makedirs(os.path.join(latest, 'dummy_dir3.0'))

            utils.symlink(latest, os.path.join(mf_path, 'active'))

            return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')

        def makePackageContent(path, pkgPath, pkgPropPath):
            pkgFile = file(pkgPath, 'w')
            for index in range(10):
                pkgFile.write(('%s%s') % (index, index))
            pkgFile.close()

            pkgFile = file(pkgPropPath, 'w')
            for index in range(10):
                pkgFile.write(('%s%s') % (index, index))
            pkgFile.close()
            uname = pylons.config['agent_user_account']
            TestCleanupController.rchown(path, uname)

        def createTestThread(serviceName):
            appGlobal = config['pylons.app_globals']
            testTh = WaitThread(appGlobal.threadMgr,
                                ServiceController.serviceCat(serviceName))
            testTh.start()
            return testTh

        def startTestProcess():
            cmd = utils.sudoCmd(["sleep", "5"],
                                pylons.config['app_user_account'])
            return Popen(cmd)

        if os.name == 'nt':
            LOG.warning(
                'Services cleanup not supported on windows. Skipping test...')
            return

        path1 = os.path.join(pylons.config['agent_root'], 'service_nodes',
                             'foo')
        path2 = os.path.join(pylons.config['agent_root'], 'service_nodes',
                             'bar')
        path3 = os.path.join(pylons.config['agent_root'], 'service_nodes',
                             'agent')

        deleteTestDir('foo')
        deleteTestDir('bar')
        deleteTestDir('agent')

        # make dirs
        makeTestDir(path1)
        makeTestDir(path2)
        makeTestDir(path3)
        all_mf, active_mf = createManifests(
            ServiceController.manifestPath('agent'))

        uname = pylons.config['agent_user_account']
        TestCleanupController.rchown(ServiceController.serviceRootPath(),
                                     uname)

        pkgDir = PackageMgr.packagePath()
        pkgPath = os.path.join(pkgDir, "foo.cronus")
        pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop")
        makePackageContent(pkgDir, pkgPath, pkgPropPath)

        # create threads
        testThFoo = createTestThread('foo')
        testThBar = createTestThread('bar')
        testThAgent = createTestThread('agent')

        # start process
        process = startTestProcess()

        # start testing
        LOG.debug('************ start cleanup')
        response = self.app.post(url(controller='cleanup', action='post'))
        LOG.debug('Delete response body = ' + response.body)
        body = json.loads(response.body)

        tm = time.time()
        while (tm + 10 > time.time()):
            response = self.app.get(body['status'], expect_errors=True)
            LOG.debug('Status response body = ' + response.body)
            body = json.loads(response.body)
            print body
            if (body['progress'] == 100):
                break
            time.sleep(0.1)

        # make sure the responses are correct
        LOG.debug('status = ' + str(response.status_int))
        assert response.status_int == 200, "HTTP response != 200"

        time.sleep(0.1)
        assert not os.path.exists(
            path1), 'service foo does exist or is not a directory'
        assert not os.path.exists(
            path2), 'service bar does exist or is not a directory'
        assert os.path.exists(
            path3), 'service agent does NOT exist or is not a directory'
        assert not testThFoo.isAlive(), 'thread Foo is still alive'
        assert not testThBar.isAlive(), 'thread Bar is still alive'
        assert not testThAgent.isAlive(), 'thread Agent is still alive'
        assert not os.path.exists(pkgPath), 'package foo exists'
        assert not os.path.exists(pkgPropPath), 'package prop foo exists'
        assert os.path.exists(pkgDir), 'package directory does not exist'
        # ensure agent cleanup is proper
        active_mf_path = ManifestController.manifestPath('agent', active_mf)
        active_link = os.path.join(ServiceController.manifestPath('agent'),
                                   'active')
        all_mf.remove(active_mf)
        actual_active_mf_path = utils.readlink(active_link)

        self.assertTrue(
            os.path.exists(active_mf_path),
            'active agent manifest got deleted but shouldn\t have')
        self.assertTrue(os.path.exists(active_link),
                        'agent active link missing')
        self.assertEqual(
            active_mf_path, actual_active_mf_path,
            'agent active link pointing to some wrong manifest; link broken?')
        for mf in all_mf:
            agnt_mf_path = ManifestController.manifestPath('agent', mf)
            self.assertFalse(
                os.path.exists(agnt_mf_path),
                'non active agent mf %s should have been deleted' % mf)
Exemplo n.º 53
0
 def __deletePackages(self):
     """ delete all packages from pakcage directory """
     package_path = PackageMgr.packagePath()
     ServiceDelete.deleteFolderContents(package_path, onlyChildren = True)