Esempio n. 1
0
    def __init__(self, passwd = None, getPerson=None,
                 host='localhost', user='******', database='jira',
                 verbose=False):

        self.getPerson=getPerson
        self.verbose=verbose
        self.sourceMap = {}
        self.sourceOwner = {}
        self.personIdMap = None
        self.productIdMap = None
        if not passwd:
            self.db = dbstore.connect("%s@%s/%s" % (user, host, database),
                                      driver = 'mysql' )
        else:
            self.db = dbstore.connect("%s:%s@%s/%s" %
                                            (user, passwd, host, database),
                                      driver = 'mysql' )

        cfg = conarycfg.ConaryConfiguration()
        client = conaryclient.ConaryClient(cfg)
        self.repos = client.getRepos()
        self.indentWrapper = textwrap.TextWrapper(
            subsequent_indent='    ',
            break_long_words=False
        )
Esempio n. 2
0
    def __init__(self,
                 passwd=None,
                 getPerson=None,
                 host='localhost',
                 user='******',
                 database='jira',
                 verbose=False):

        self.getPerson = getPerson
        self.verbose = verbose
        self.sourceMap = {}
        self.sourceOwner = {}
        self.personIdMap = None
        self.productIdMap = None
        if not passwd:
            self.db = dbstore.connect("%s@%s/%s" % (user, host, database),
                                      driver='mysql')
        else:
            self.db = dbstore.connect("%s:%s@%s/%s" %
                                      (user, passwd, host, database),
                                      driver='mysql')

        cfg = conarycfg.ConaryConfiguration()
        client = conaryclient.ConaryClient(cfg)
        self.repos = client.getRepos()
        self.indentWrapper = textwrap.TextWrapper(subsequent_indent='    ',
                                                  break_long_words=False)
Esempio n. 3
0
    def create(self, name):
        path = self._getTemplate()[1] % 'postgres'
        db = dbstore.connect(path, self.driver)

        dbName = self.translate(name)

        cu = db.cursor()
        # this check should never be required outside of the test suite,
        # and it could be kind of dangerous being called in production.
        cu.execute("SELECT datname FROM pg_database")
        createDb = True
        if dbName in [x[0] for x in cu.fetchall()]:
            createDb = False
            db.close()
            if self.cfg.debugMode:
                import gc
                while gc.collect():
                    pass

                reposDb = dbstore.connect(
                        self._getTemplate()[1] % dbName, self.driver)
                reposDb.loadSchema()
                reposCu = reposDb.cursor()
                for t in reposDb.tempTables:
                    reposCu.execute("DROP TABLE %s" % (t,))
                for t in reposDb.tables:
                    reposCu.execute("DROP TABLE %s CASCADE" % (t,))
                reposDb.close()
            else:
                raise RepositoryAlreadyExists(name)
            
        if createDb:
            cu.execute("CREATE DATABASE %s %s" % (dbName, self.tableOpts))
        db.close()
        RepositoryDatabase.create(self, name)
Esempio n. 4
0
def conaryrc(req):
    hostname = req.host
    cfg = req.cfg
    if cfg.mirrorsInGroup:
        return managedConaryrc(req)
    if hostname.endswith(':80') or hostname.endswith(':443'):
        hostname = hostname.split(':')[0]

    # Lack of a repository indicates proxy mode (RBM-273)
    if cfg.repositoryDB == None:
        body = "proxyMap * conarys://%s\n" % hostname
    else:
        # Otherwise, we're in mirror mode
        db = dbstore.connect(cfg.repositoryDB[1], driver = cfg.repositoryDB[0])
        cu = db.cursor()
        cu.execute("""SELECT LABEL FROM Labels WHERE EXISTS
                          ( SELECT labelId FROM LabelMap
                                JOIN Nodes USING(itemId, branchId)
                                JOIN Instances USING(itemId, versionId)
                            WHERE isPresent=1
                                AND LabelMap.labelId = Labels.labelId )""")
        serverNames = set()
        for label, in cu:
            if '@' not in label:
                continue
            name = label.split('@', 1)[0]
            serverNames.add(name)

        body = ""
        for name in serverNames:
            body += "repositoryMap %s https://%s/conary/\n" % (name, hostname)

    req.response.content_type = 'text/plain'
    req.response.body = body
    return req.response
Esempio n. 5
0
def main(args):
    if len(args) != 1:
        sys.exit("Usage: %s <psql db path>" % (sys.argv[0],))
    path = args[0]

    db = dbstore.connect(path, 'postgresql')
    db.loadSchema()
    cu = db.cursor()

    tableDeps = {}
    for table in db.tables:
        if table.lower == 'databaseversion':
            continue
        tableDeps[table] = set()

    for table in db.tables:
        cu.execute("""select b.relname
            from pg_catalog.pg_class a
            join pg_catalog.pg_constraint r on a.oid = r.conrelid
            left join pg_catalog.pg_class b on r.confrelid = b.oid
            where a.relname = ? and r.contype = 'f'""", table)
        tableDeps[table] = set(x[0] for x in cu)

    ordered = []
    while tableDeps:
        for table, deps in tableDeps.items():
            if deps:
                continue
            print table
            ordered.append(table)
            for otherDeps in tableDeps.values():
                otherDeps.discard(table)
            del tableDeps[table]

    print repr(ordered)
Esempio n. 6
0
def getDB(dbfile, create=False):
    db = dbstore.connect(dbfile, driver="sqlite")
    if create:
        createSchema(db)
    v = db.getVersion()
    assert (v == DB_VERSION)
    return db
Esempio n. 7
0
    def fixtureImageJob(self, cfg):
        """
        ImageJob fixture.

        Creates the following setup:
            - One user:
                - test (a basic user with no special privileges)
            - A project called "foo"
                - "test" is a member of "foo"
                - A build called "Test Build"
                - A single image job, in the "started" state

        @param cfg: The current effective Mint configuration.
        @return: A 2-tuple consisting of the current Mint configuration and a
            a dictionary containing the following:
                - C{test} - the id of the user "test"
        """
        db = dbstore.connect(cfg.dbPath, cfg.dbDriver)
        testId = self.createUser(cfg, db, 'test')

        client = shimclient.ShimMintClient(cfg, ('test', 'testpass'))

        projectId = client.newProject("Foo", "foo", "rpath.org")

        build = client.newBuild(projectId, "Test Build")
        build.setBuildType(buildtypes.STUB_IMAGE)

        stockBuildFlavor(db, build.getId())

        prodJob = client.startImageJob(build.getId())
        return cfg, { 'test': testId }
Esempio n. 8
0
 def connect(self):
     if self.db:
         return self.db
     if self.verbose:
         print "CONNECT DATABASE", self.driver, self.path
     self.db = dbstore.connect(self.path, driver = self.driver)
     return self.db
Esempio n. 9
0
    def getMirrorAcl(self, project, username):
        """
        Given a project and a username, will determine whether or not the
        user has the ability to mirror.
        @param project: the project to check against
        @param username: the user whose credentials should be checked
        @returns: C{True} if C{username} can mirror C{project}; C{False}
            otherwise.
        """
        dbCon = project.server._server.projects.reposDB.getRepositoryDB( \
            project.getFQDN())
        db = dbstore.connect(dbCon[1], dbCon[0])

        cu = db.cursor()

        cu.execute("""SELECT canMirror
                          FROM Users
                          LEFT JOIN UserGroupMembers ON Users.userId =
                                  UserGroupMembers.userId
                          LEFT JOIN UserGroups ON UserGroups.userGroupId =
                                  UserGroupMembers.userGroupId
                          WHERE Users.username=?""", username)

        try:
            # nonexistent results trigger value error
            canMirror = max([x[0] for x in cu.fetchall()])
        except ValueError:
            canMirror = None
        db.close()
        return canMirror
Esempio n. 10
0
def getDB(dbfile, create = False):
    db = dbstore.connect(dbfile, driver="sqlite")
    if create:
        createSchema(db)
    v = db.getVersion()
    assert (v == DB_VERSION)
    return db
Esempio n. 11
0
 def testTransaction(self):
     hdb = self.harness.getDB()
     db = hdb.connect()
     db2 = dbstore.connect(hdb.path, hdb.driver)
     db2.transaction()
     self.assertTrue(sqlerrors.DatabaseLocked, db.transaction)
     db2.commit()
     db.transaction()
Esempio n. 12
0
 def getDB(self):
     if self._db:
         return self._db
     (driver, database) = self.cfg.repositoryDB
     self._db = dbstore.connect(database, driver)
     schema.setupTempTables(self._db)
     depSchema.setupTempDepTables(self._db)
     return self._db
Esempio n. 13
0
    def delete(self, name):
        path = self._getTemplate()[1] % 'mysql'
        db = dbstore.connect(path, 'mysql')
        reposName = self.translate(name)

        cu = db.cursor()
        cu.execute("DROP DATABASE %s" % reposName)
        util.rmtree(path + reposName, ignore_errors = True)
Esempio n. 14
0
 def testTransaction(self):
     hdb = self.harness.getDB()
     db = hdb.connect()
     db2 = dbstore.connect(hdb.path, hdb.driver)
     db2.transaction()
     self.assertTrue(sqlerrors.DatabaseLocked, db.transaction)
     db2.commit()
     db.transaction()
Esempio n. 15
0
 def getDB(self):
     if self._db:
         return self._db
     (driver, database) = self.cfg.repositoryDB
     self._db = dbstore.connect(database, driver)
     schema.setupTempTables(self._db)
     depSchema.setupTempDepTables(self._db)
     return self._db
Esempio n. 16
0
 def handle(self, *args, **options):
     dbVersion = models.DatabaseVersion(
         version=schema.RBUILDER_DB_VERSION.major,
         minor=schema.RBUILDER_DB_VERSION.minor)
     dbVersion.save()
     db = dbstore.connect(settings.DATABASES['default']['NAME'], 'sqlite')
     version = schema.loadSchema(db, should_migrate=True)
     print "Migrated rBuilder schema to %s" % version
Esempio n. 17
0
    def action(self):
        db = dbstore.connect(self.cfg.dbPath, self.cfg.dbDriver)
        if self.options.create:
            print >> sys.stderr, "Force-creating database schema ..."
            db.loadSchema()
            schema.createSchema(db, cfg=self.cfg)
        else:
            schema.loadSchema(db, self.cfg, self.options.should_migrate)

        return 0
Esempio n. 18
0
 def testSqliteRegexp(self):
     db = dbstore.connect(':memory:', 'sqlite')
     cu = db.cursor()
     cu.execute('create table foo(bar varchar)')
     cu.execute("insert into foo values ('foo1')")
     cu.execute("insert into foo values ('foo2')")
     cu.execute("insert into foo values ('foo3')")
     cu.execute('select * from foo where bar regexp ".*2"')
     matches = cu.fetchall()
     assert(matches == [ ('foo2',) ])
Esempio n. 19
0
 def testSqliteRegexp(self):
     db = dbstore.connect(':memory:', 'sqlite')
     cu = db.cursor()
     cu.execute('create table foo(bar varchar)')
     cu.execute("insert into foo values ('foo1')")
     cu.execute("insert into foo values ('foo2')")
     cu.execute("insert into foo values ('foo3')")
     cu.execute('select * from foo where bar regexp ".*2"')
     matches = cu.fetchall()
     assert (matches == [('foo2', )])
Esempio n. 20
0
def startRepository(cfg, logger=None):
    reposDir = cfg.getReposDir()
    util.mkdirChain(reposDir)

    # Generate and store a random password for the rmake user if not configured
    # with one.
    if not cfg.reposUser:
        passwordFile = reposDir + '/password'
        if os.path.exists(passwordFile):
            password = open(passwordFile).readline().strip()
        else:
            password = ''.join([ chr(random.randrange(ord('a'), ord('z'))) 
                               for x in range(10)])
            open(passwordFile, 'w').write(password + '\n')
            os.chmod(reposDir + '/password', 0700)
        cfg.reposUser.addServerGlob(cfg.reposName, 'rmake', password)

    serverCfg = cny_server.ServerConfig()
    serverCfg.repositoryDB = ('sqlite', cfg.getReposDbPath())
    serverCfg.contentsDir = cfg.getContentsPath()
    serverCfg.port = cfg.getReposInfo()[1]
    serverCfg.configKey('serverName', cfg.reposName)

    # Transfer SSL settings from rMake config object
    if getattr(cny_server, 'SSL', None):
        # The server supports starting in SSL mode
        serverCfg.useSSL = cfg.reposRequiresSsl()
        serverCfg.sslCert = cfg.sslCertPath
        serverCfg.sslKey = cfg.sslCertPath
    elif cfg.reposRequiresSsl():
        raise errors.RmakeError("Tried to start repository at %s, but missing "
                "ssl server library: Please install m2crypto." %
                (cfg.getRepositoryUrl(),))

    (driver, database) = serverCfg.repositoryDB
    db = dbstore.connect(database, driver)

    # Note - this will automatically migrate this repository! 
    # Since this is a throwaway repos anyway, I think that's
    # acceptable.
    cny_schema.loadSchema(db, doMigrate=True)
    db.commit()
    db.close()

    user, password = cfg.reposUser.find(cfg.reposName)
    addUser(serverCfg, user, password, write=True)
    if not serverCfg.useSSL:
        # allow anonymous access if we're not securing this repos
        # by using SSL - no use securing access if it's all going to be
        # viewable via tcpdump.
        addUser(serverCfg, 'anonymous', 'anonymous')

    return _startServer(serverCfg, cfg.getReposLogPath(),
            cfg.getReposConfigPath(), 'repository')
Esempio n. 21
0
    def action(self, fqdn=None):
        self.log.info("Updating package index")

        self.db = dbstore.connect(self.cfg.dbPath, driver=self.cfg.dbDriver)
        self.db.connect()
        self.db.loadSchema()
        cu = self.db.cursor()
        labelsTable = projects.LabelsTable(self.db, self.cfg)
        self.db.commit()

        cu = self.db.cursor()
        sql = """SELECT projectId, fqdn, EXISTS(SELECT * FROM InboundMirrors
                           WHERE projectId=targetProjectId) AS localMirror
                         FROM Projects
                         WHERE external AND NOT hidden AND NOT disabled"""
        args = []
        if fqdn:
            sql += " AND fqdn = ?"
            args.append(fqdn)
        cu.execute(sql, args)

        labels = {}
        projectIds = {}
        netclients = {}
        hasErrors = False

        for projectId, hostname, localMirror in cu.fetchall():
            try:
                self.log.info("Retrieving labels from %s...", hostname)
                l, repMap, userMap, entMap = labelsTable.getLabelsForProject(projectId)

                hostname = repMap.keys()[0]
                labels[hostname] = versions.Label(l.keys()[0])
                projectIds[hostname] = projectId

                ccfg = conarycfg.ConaryConfiguration(False)
                ccfg.configLine("proxyMap * conarys://localhost")
                ccfg.root = ccfg.dbPath = ":memory:"
                ccfg.repositoryMap = repMap
                if not localMirror:
                    for host, authInfo in userMap:
                        ccfg.user.addServerGlob(host, authInfo[0], authInfo[1])
                    for host, entitlement in entMap:
                        ccfg.entitlement.addEntitlement(host, entitlement[1])
                ccfg = helperfuncs.configureClientProxies(
                    ccfg, self.cfg.useInternalConaryProxy, self.cfg.proxy, self.cfg.getInternalProxies()
                )
                repos = conaryclient.ConaryClient(ccfg).getRepos()
                netclients[hostname] = repos
            except Exception, e:
                self.log.error("Exception from %s", hostname)
                self.log.error(str(e))
                hasErrors = True
Esempio n. 22
0
    def newMintCfg(self, name):
        cfg = FixtureCache.newMintCfg(self, name)
        cfg.dbDriver = 'sqlite'
        cfg.dbPath = os.path.join(cfg.dataPath, 'mintdb')

        reposDBPath = os.path.join(cfg.dataPath, 'repos', '%s', 'sqldb')
        cfg.configLine('database default sqlite ' + reposDBPath)

        from mint.db import schema
        db = dbstore.connect(cfg.dbPath, cfg.dbDriver)
        schema.loadSchema(db, cfg)

        return cfg
Esempio n. 23
0
    def getAdminAcl(self, project, username):
        dbCon = project.server._server.projects.reposDB.getRepositoryDB( \
            project.getFQDN())
        db = dbstore.connect(dbCon[1], dbCon[0])

        cu = db.cursor()
        cu.execute("""SELECT MAX(admin) FROM Users
                      JOIN UserGroupMembers ON
                          Users.userId = UserGroupMembers.userId
                      JOIN UserGroups ON
                          UserGroups.userGroupId = UserGroupMembers.userGroupId
                      WHERE Users.username=?""", username)
        return cu.fetchone()[0]
Esempio n. 24
0
    def migrate(self):
        import tempfile
        import os
        from conary import dbstore

        # figure out where the database lives currently
        assert(self.db.driver == 'sqlite')
        dbPath = self.db.database
        assert(isinstance(dbPath, str))
        # make a new database file
        fd, fn = tempfile.mkstemp(prefix=os.path.basename(dbPath) + '-new-',
                                  dir=os.path.dirname(dbPath))
        os.close(fd)
        newdb = dbstore.connect(fn, driver='sqlite')
        # create the schema in the new db
        newdb.loadSchema()
        createSchema(newdb)
        # make sure we have a good view of the new schema
        newdb.commit()
        newdb.loadSchema()

        cu = self.cu
        # have to commit in order to attach
        self.db.commit()
        cu.execute("ATTACH '%s' AS newdb" %fn, start_transaction=False)

        for t in newdb.tables.keys():
            self.message('Converting database schema to version 20 '
                         '- current table: %s' %t)
            cu.execute('INSERT OR REPLACE INTO newdb.%s '
                       'SELECT * FROM %s' % (t, t))

        # fix up some potentially bad entries we know about
        cu.execute("""UPDATE newdb.TroveInfo
                      SET data='1.0'
                      WHERE hex(data)='31' AND infotype=3""")
        cu.execute("""UPDATE newdb.Dependencies
                      SET flag='1.0'
                      WHERE name LIKE 'conary:%' AND flag='1'""");

        self.message('Converting database schema to version 20 '
                     '- committing')
        self.db.commit()
        self.message('')
        newdb.close()
        os.chmod(fn, 0644)
        os.rename(dbPath, dbPath + '-pre-schema-update')
        os.rename(fn, dbPath)
        self.db.reopen()
        self.db.loadSchema()
        return self.Version
Esempio n. 25
0
 def testMigrateAll(self):
     dbPath = self.workDir + '/jobs.db'
     shutil.copyfile(resources.get_archive('jobs.db.v1'))
     db = dbstore.connect(dbPath, driver = "sqlite", timeout=10000)
     db.loadSchema()
     cu = db.cursor()
     assert(cu.execute('select state from Jobs where jobId=1').next()[0] == -1)
     assert(cu.execute('select state from Jobs where jobId=3').next()[0] == 99)
     mgr = schema.SchemaManager(db)
     m = schema.Migrator(db, mgr)
     m.migrate(1, schema.SCHEMA_VERSION)
     db.loadSchema()
     assert(cu.execute('select state from Jobs where jobId=1').next()[0] == 1)
     assert(cu.execute('select state from Jobs where jobId=3').next()[0] == 5)
Esempio n. 26
0
    def _openDb(self):
        if not self._db:
            self._db = dbstore.connect(self._cfg.dbPath, self._cfg.dbDriver)
            self._autoDb = True

        # check to make sure the schema version is correct
        try:
            schema.checkVersion(self._db)
        except sqlerrors.SchemaVersionError:
            rethrow(mint_error.DatabaseVersionMismatch, False)

        tables = TableCache(self._db, self._cfg)
        self._copyTables(tables)
        if self._db.inTransaction(True):
            self._db.commit()
Esempio n. 27
0
    def testDBInstances(self):
        cx = dbstore.connect(":memory:", driver="sqlite")
        cx.loadSchema()
        idb = sqldb.DBInstanceTable(cx)
        idb.addId('fred', 1, 2, [1, 2])
        assert(idb[('fred', 1, 2)] == 1)
        assert(idb.getId(1) == ('fred', 1,2,1) )
        assert(idb.get(('fred', 1, 2), None) == 1)

        idb.addId('wilma', 5, 6, [1, 2])
        assert(idb[('wilma', 5, 6)] == 2)
        idb.delId(2)
        self.assertRaises(KeyError, idb.__getitem__, ('wilma', 5, 6))
        idb.delId(1)
        assert(idb.get(('fred', 1, 2), None) == None)
Esempio n. 28
0
def getDb(path):
    if os.path.exists(path):
        print "Deleting database"
        os.remove(path)
    if os.listdir("/srv/rmake-repos/contents/"):
        print "Deleting contents..."
        os.system("rm -rf /srv/rmake-repos/contents/*")
    open(path, 'w')
    os.chown(path, pwd.getpwnam('apache').pw_uid,
                   pwd.getpwnam('apache').pw_gid)

    db = dbstore.connect(path, driver='sqlite')
    schema.loadSchema(db, True)
    schema.setupTempTables(db)
    return db
Esempio n. 29
0
    def getWriteAcl(self, project, username):
        dbCon = project.server._server.projects.reposDB.getRepositoryDB( \
            project.getFQDN())
        db = dbstore.connect(dbCon[1], dbCon[0])

        cu = db.cursor()

        cu.execute("""SELECT MAX(canWrite)
                          FROM Users
                          LEFT JOIN UserGroupMembers ON Users.userId =
                                  UserGroupMembers.userId
                          LEFT JOIN Permissions ON Permissions.userGroupId =
                                  UserGroupMembers.userGroupId
                          WHERE Users.username=?""", username)
        return cu.fetchone()[0]
Esempio n. 30
0
    def testDBInstances(self):
        cx = dbstore.connect(":memory:", driver="sqlite")
        cx.loadSchema()
        idb = sqldb.DBInstanceTable(cx)
        idb.addId('fred', 1, 2, [1, 2])
        assert (idb[('fred', 1, 2)] == 1)
        assert (idb.getId(1) == ('fred', 1, 2, 1))
        assert (idb.get(('fred', 1, 2), None) == 1)

        idb.addId('wilma', 5, 6, [1, 2])
        assert (idb[('wilma', 5, 6)] == 2)
        idb.delId(2)
        self.assertRaises(KeyError, idb.__getitem__, ('wilma', 5, 6))
        idb.delId(1)
        assert (idb.get(('fred', 1, 2), None) == None)
Esempio n. 31
0
def getDb(path):
    if os.path.exists(path):
        print "Deleting database"
        os.remove(path)
    if os.listdir("/srv/rmake-repos/contents/"):
        print "Deleting contents..."
        os.system("rm -rf /srv/rmake-repos/contents/*")
    open(path, 'w')
    os.chown(path,
             pwd.getpwnam('apache').pw_uid,
             pwd.getpwnam('apache').pw_gid)

    db = dbstore.connect(path, driver='sqlite')
    schema.loadSchema(db, True)
    schema.setupTempTables(db)
    return db
Esempio n. 32
0
    def testDBTroveFiles(self):
        cx = dbstore.connect(":memory:", driver="sqlite")
        cx.loadSchema()
        fs = sqldb.DBTroveFiles(cx)
        cu = cx.cursor()
        fs.addItem(cu, self.id1, 1, "/bin/ls", self.fid1, 11, "abc",
                   ["tag1", "tag2"])
        fs.addItem(cu, self.id2, 2, "/bin/cat", self.fid2, 11, "def", ["tag1"])
        fs.addItem(cu, self.id3, 1, "/bin/dd", self.fid3, 12, "tuv", [])
        fs.addItem(cu, self.id4, 2, "/bin/bc", self.fid4, 12, "xyz", [])
        assert ([x for x in fs[11]] == [("/bin/ls", "abc"),
                                        ("/bin/cat", "def")])
        assert ([x for x in fs[12]] == [("/bin/dd", "tuv"),
                                        ("/bin/bc", "xyz")])
        assert ([x for x in fs.iterFilesWithTag('tag1')
                 ] == ['/bin/cat', '/bin/ls'])
        assert ([x for x in fs.iterFilesWithTag('tag2')] == ['/bin/ls'])

        fs.delInstance(11)
        assert ([x for x in fs[11]] == [])

        # make sure the tags are gone
        assert ([x for x in fs.iterFilesWithTag('tag1')] == [])
        cu = cx.cursor()
        cu.execute('select * from DBFileTags')
        assert ([x for x in cu] == [])

        assert (fs.getFileByFileId(self.fid3, 0) == ("/bin/dd", "tuv"))
        self.assertRaises(KeyError, fs.getFileByFileId, self.fid7, 0)

        assert ([x for x in fs[12]] == [("/bin/dd", "tuv"),
                                        ("/bin/bc", "xyz")])
        fs.removePath(12, "/bin/dd")
        assert ([x for x in fs.getByInstanceId(12)] == [("/bin/bc", "xyz")])
        assert (fs.getFileByFileId(self.fid4,
                                   justPresent=False) == ('/bin/bc', 'xyz'))
        assert ([x for x in fs.getByInstanceId(12, justPresent=False)
                 ] == [("/bin/dd", "tuv"), ("/bin/bc", "xyz")])
        fs.delInstance(12)

        fs.addItem(cu, self.id1, 1, "/bin/ls", self.fid1, 11, "abc", [])
        fs.addItem(cu, self.id2, 2, "/bin/cat", self.fid2, 11, "def", [])
        fs.addItem(cu, self.id3, 1, "/bin/dd", self.fid3, 11, "tuv", [])
        fs.addItem(cu, self.id4, 2, "/bin/bc", self.fid4, 11, "xyz", [])
        assert ([x
                 for x in fs[11]] == [("/bin/ls", "abc"), ("/bin/cat", "def"),
                                      ("/bin/dd", "tuv"), ("/bin/bc", "xyz")])
Esempio n. 33
0
    def testDBTroveFiles(self):
        cx = dbstore.connect(":memory:", driver="sqlite")
        cx.loadSchema()
        fs = sqldb.DBTroveFiles(cx)
        cu = cx.cursor()
        fs.addItem(cu, self.id1, 1, "/bin/ls", self.fid1, 11, "abc", [ "tag1", "tag2"])
        fs.addItem(cu, self.id2, 2, "/bin/cat", self.fid2, 11, "def", ["tag1"])
        fs.addItem(cu, self.id3, 1, "/bin/dd", self.fid3, 12, "tuv", [])
        fs.addItem(cu, self.id4, 2, "/bin/bc", self.fid4, 12, "xyz", [])
        assert([x for x  in fs[11]] == [("/bin/ls", "abc"),
                                        ("/bin/cat", "def")])
        assert([x for x  in fs[12]] == [("/bin/dd", "tuv"),
                                        ("/bin/bc", "xyz")])
        assert([x for x in fs.iterFilesWithTag('tag1') ] == 
                                    ['/bin/cat', '/bin/ls'] )
        assert([x for x in fs.iterFilesWithTag('tag2') ] == ['/bin/ls' ])
        
        fs.delInstance(11)
        assert([x for x  in fs[11]] == [])
        
        # make sure the tags are gone
        assert([x for x in fs.iterFilesWithTag('tag1') ] == [])
        cu = cx.cursor()
        cu.execute('select * from DBFileTags')
        assert([x for x in cu ] == [])

        assert(fs.getFileByFileId(self.fid3, 0) == ("/bin/dd", "tuv"))
        self.assertRaises(KeyError, fs.getFileByFileId, self.fid7, 0)

        assert([x for x  in fs[12]] == [("/bin/dd", "tuv"),
                                        ("/bin/bc", "xyz")])
        fs.removePath(12, "/bin/dd")
        assert([x for x  in fs.getByInstanceId(12)] == [("/bin/bc", "xyz")])
        assert(fs.getFileByFileId(self.fid4, justPresent = False) == 
                ('/bin/bc', 'xyz'))
        assert([x for x  in fs.getByInstanceId(12, justPresent = False)] == 
                [("/bin/dd", "tuv"), ("/bin/bc", "xyz")])
        fs.delInstance(12)

        fs.addItem(cu, self.id1, 1, "/bin/ls", self.fid1, 11, "abc", [])
        fs.addItem(cu, self.id2, 2, "/bin/cat", self.fid2, 11, "def", [])
        fs.addItem(cu, self.id3, 1, "/bin/dd", self.fid3, 11, "tuv", [])
        fs.addItem(cu, self.id4, 2, "/bin/bc", self.fid4, 11, "xyz", [])
        assert([x for x  in fs[11]] == [("/bin/ls", "abc"),
                                        ("/bin/cat", "def"),
                                        ("/bin/dd", "tuv"),
                                        ("/bin/bc", "xyz")])
Esempio n. 34
0
    def loadFixture(self, name, loadFn=None):
        """
        Loads the fixture for the unit test.
        @param name: the name of the fixture (e.g. "Full")
        @returns: A 3-typle consisting of a database connection,
           a Mint client with admin credentials, and a dictionary of 
           fixture data (may be empty).
        """

        # reset the cached db connection
        mint.db.database.dbConnection = None
        self.cfg, fixtureData = fixtureCache.load(name, loadFn=loadFn)
        db = dbstore.connect(self.cfg.dbPath, self.cfg.dbDriver)
        # this is so fugly it makes me wanna cry --gafton
        mint.db.database.dbConnection = db
        mint.db.database.tables = None
        return db, fixtureData
Esempio n. 35
0
 def connect(self, dbName):
     if not self.check():
         raise RuntimeError("postmaster is dead")
     start = time.time()
     while True:
         try:
             return dbstore.connect('conary@localhost:%s/%s'
                     % (self.port, dbName), 'postgresql')
         except sqlerrors.DatabaseError, err:
             if ('the database system is starting up' in err.msg
                     or 'Connection refused' in err.msg):
                 if time.time() - start > 15:
                     self.kill()
                     raise RuntimeError("Database did not start")
                 time.sleep(0.1)
                 continue
             raise
Esempio n. 36
0
 def testMigrateAll(self):
     dbPath = self.workDir + '/jobs.db'
     shutil.copyfile(resources.get_archive('jobs.db.v1'))
     db = dbstore.connect(dbPath, driver="sqlite", timeout=10000)
     db.loadSchema()
     cu = db.cursor()
     assert (
         cu.execute('select state from Jobs where jobId=1').next()[0] == -1)
     assert (
         cu.execute('select state from Jobs where jobId=3').next()[0] == 99)
     mgr = schema.SchemaManager(db)
     m = schema.Migrator(db, mgr)
     m.migrate(1, schema.SCHEMA_VERSION)
     db.loadSchema()
     assert (
         cu.execute('select state from Jobs where jobId=1').next()[0] == 1)
     assert (
         cu.execute('select state from Jobs where jobId=3').next()[0] == 5)
Esempio n. 37
0
    def create(self, name):
        path = self._getTemplate()[1] % 'mysql'
        db = dbstore.connect(path, 'mysql')

        dbName = self.translate(name)

        cu = db.cursor()
        # this check should never be required outside of the test suite,
        # and it could be kind of dangerous being called in production.
        # audited for SQL injection
        cu.execute("SHOW DATABASES")
        if dbName in [x[0] for x in cu.fetchall()]:
            if self.cfg.debugMode:
                cu.execute("DROP DATABASE %s" % dbName)
            else:
                raise RepositoryAlreadyExists(name)
        cu.execute("CREATE DATABASE %s %s" % (dbName, self.tableOpts))
        db.close()
        RepositoryDatabase.create(self, name)
Esempio n. 38
0
    def fixtureEmpty(self, cfg):
        """
        Empty fixture. Should be used when you want a (mostly) blank setup
        for testing.

        Creates the following setup:
            - Two users:
                - test (a basic user with no special privileges)
                - admin (a user wih admin privileges)

        @param cfg: The current effective Mint configuration.
        @return: A 2-tuple consisting of the current Mint configuration and a
            a dictionary containing the following:
                - C{test} - the id of the user "test"
                - C{admin} - the id of the user "admin"
        """
        db = dbstore.connect(cfg.dbPath, cfg.dbDriver)

        testId = self.createUser(cfg, db, "test")
        adminId = self.createUser(cfg, db, "admin", isAdmin=True)

        return cfg, { 'test': testId, 'admin': adminId }
Esempio n. 39
0
    def _openClient(self, root):
        self.conarycfg.root = root
        # page_size has to be set before the first table is created
        path = util.joinPaths(root, self.conarycfg.dbPath + "/conarydb")
        util.mkdirChain(os.path.dirname(path))
        db = dbstore.connect(path, driver="sqlite")
        cu = db.cursor()
        cu.execute("PRAGMA page_size = 4096")
        db.commit()
        cu.execute("VACUUM")
        db.commit()
        db.close()

        # The rest are per-session and apply only to this install job
        cclient = conaryclient.ConaryClient(self.conarycfg)
        cclient.db.opJournalPath = None
        db = cclient.db.db.db
        cu = db.cursor()
        cu.execute("PRAGMA cache_size = 200000")
        cu.execute("PRAGMA journal_mode = MEMORY")
        db.commit()
        return cclient
Esempio n. 40
0
    def __init__(self, db=None, driver=None, path=None):
        cmd.Cmd.__init__(self)

        # default to .head off
        self.show_headers = False
        # default to .mode list
        self.format = self.format_list
        # a dictionary of column number: width for manual setting
        self.manual_widths = {}
        # use a pager?
        self.use_pager = False
        # calculate column widths for column view?
        self.auto_width = False
        # display stats: N rows in set (0.00 sec)
        self.show_stats = True

        if driver and path:
            self.db = dbstore.connect(path, driver=driver)
        elif db:
            self.db = db
        else:
            raise RuntimeError, 'driver and path OR db must be given'
        self.cu = self.db.cursor()
Esempio n. 41
0
    def _testSlowActionWithStandalone(self, useSSL = False):
        # Test to make sure that slow commits still work even with the
        # proxy keepalive code added (CNY-1341)
        cfg = server.ServerConfig()
        cfg.port = testhelp.findPorts(1)[0]
        cfg.contentsDir = ('legacy', [self.workDir + '/contents'])
        cfg.repositoryDB = ('sqlite', self.workDir + '/serverdb')
        cfg.logFile = self.workDir + '/serverlog'
        cfg.tmpDir = self.workDir + '/tmp'
        cfg.serverName = 'localhost'
        util.mkdirChain(cfg.tmpDir)
        util.mkdirChain(cfg.contentsDir[1][0])
        if useSSL:
            cfg.useSSL = True
            cfg.sslCert = os.path.join(resources.get_archive(), 'ssl-cert.crt')
            cfg.sslKey = os.path.join(resources.get_archive(), 'ssl-cert.key')

        (driver, database) = cfg.repositoryDB
        db = dbstore.connect(database, driver)
        schema.loadSchema(db)
        schema.setupTempTables(db)
        auth = netauth.NetworkAuthorization(db, 'localhost')
        auth.addRole('foo')
        auth.addUser('foo', 'foo')
        auth.addRoleMember('foo', 'foo')
        auth.addAcl('foo', None, None, write = True)


        if useSSL:
            proto = "https"
        else:
            proto = "http"
        baseUrl = '%s://localhost:%s/' % (proto, cfg.port)

        pid = os.fork()
        if not pid:
            try:
                netServer = netserver.NetworkRepositoryServer(cfg, baseUrl)

                oldGetChangeSet = netServer.getChangeSet
                @netserver.accessReadOnly
                def getChangeSet(*args, **kw):
                    rv = oldGetChangeSet(*args, **kw)
                    # make sure the client sends its message
                    self.sleep(7)
                    return rv
                getChangeSet.im_func = getChangeSet
                netServer.getChangeSet = getChangeSet

                class HttpRequestsSubclass(server.HttpRequests):
                    tmpDir = cfg.tmpDir
                    netRepos = proxy.SimpleRepositoryFilter(cfg, baseUrl, netServer)
                    restHandler = None

                HttpRequestsSubclass.cfg = cfg

                if useSSL:
                    ctx = server.createSSLContext(cfg)
                    httpServer = server.SecureHTTPServer(("", cfg.port),
                                                         HttpRequestsSubclass, ctx)
                else:
                    httpServer = server.HTTPServer(("", cfg.port),
                                                   HttpRequestsSubclass)
                self.captureOutput(server.serve, httpServer)
            finally:
                os._exit(0)
        try:
            sock_utils.tryConnect("127.0.0.1", cfg.port)
            cfg = conarycfg.ConaryConfiguration(False)
            cfg.repositoryMap  = {'localhost' : baseUrl }
            cfg.user.addServerGlob('localhost', 'foo', 'foo')
            client = conaryclient.ConaryClient(cfg)
            repos = client.getRepos()

            trv, cs = self.Component('foo:run', '1')
            repos.commitChangeSet(cs)
            # getTrove will fail because it takes more than 5 seconds
            assert(repos.getTrove(*trv.getNameVersionFlavor()))
        finally:
            os.kill(pid, signal.SIGTERM)
            os.waitpid(pid, 0)
Esempio n. 42
0
                                                argv=argv)
    except options.OptionError, msg:
        print >> sys.stderr, msg
        sys.exit(1)

    if "help" in argSet:
        usage(argv[0])
        sys.exit(0)

    startLogging()

    if not cfg.check():
        raise RuntimeError("configuration file is invalid")

    (driver, database) = cfg.repositoryDB
    db = dbstore.connect(database, driver)
    # if there is no schema or we're asked to migrate, loadSchema
    dbVersion = db.getVersion()
    # a more recent major is not compatible
    if dbVersion.major > schema.VERSION.major:
        log.error("code base too old for this repository database")
        log.error("repo=%s code=%s", dbVersion, schema.VERSION)
        sys.exit(-1)
    db.close()
    return (cfg, argSet, otherArgs[1:])


def main():
    opts = {}
    opts["fix"] = options.NO_PARAM
    cfg, opts, args = getServer(opts)
Esempio n. 43
0
 def __init__(self, driver, db, verbose=True):
     self.db = dbstore.connect(db, driver)
     self.driver = driver
     self.db.loadSchema()
     self.verbose = verbose
     self._hint = ''
Esempio n. 44
0
 def reopen_fork(self, forked=False):
     self.db.close()
     self.db = dbstore.connect(self.cfg.dbPath, self.cfg.dbDriver)
     self.users.db = self.db
Esempio n. 45
0
 def __init__(self, cfg, db = None):
     self.cfg = cfg
     self.db = dbstore.connect(cfg.dbPath, cfg.dbDriver)
     self.users = self.Users(self.db)
Esempio n. 46
0
 def testSqlDataStore(self):
     db = dbstore.connect(':memory:', driver='sqlite')
     db.loadSchema()
     self._testDataStore(SqlDataStore(db))
Esempio n. 47
0
 def open(self):
     return dbstore.connect(self.dbpath,
                            driver=self.driver,
                            timeout=120000,
                            lockJournal=True)
Esempio n. 48
0
def startRepository(cfg, fork=True, logger=None):
    global conaryDir
    baseDir = cfg.serverDir
    if logger is None:
        logger = log

    reposDir = '%s/repos' % baseDir
    util.mkdirChain(reposDir)
    if not cfg.reposUser:
        passwordFile = reposDir + '/password'
        if os.path.exists(passwordFile):
            password = open(passwordFile).readline()[:-1]
        else:
            password = ''.join(
                [chr(random.randrange(ord('a'), ord('z'))) for x in range(10)])
            open(passwordFile, 'w').write(password + '\n')
            os.chmod(reposDir + '/password', 0700)
        cfg.reposUser.addServerGlob(cfg.reposName, 'rmake', password)

    serverConfig = os.path.join(cfg.getReposDir(), 'serverrc')
    if os.path.exists(serverConfig):
        os.unlink(serverConfig)
    serverCfg = server.ServerConfig(os.path.join(cfg.getReposDir(),
                                                 'serverrc'))
    serverCfg.repositoryDB = ('sqlite', cfg.getReposDbPath())
    serverCfg.contentsDir = cfg.getContentsPath()
    serverCfg.port = cfg.getReposInfo()[1]
    serverCfg.configKey('serverName', cfg.reposName)  # this works with either
    # 1.0.16 or 1.0.17+
    serverCfg.logFile = cfg.getReposDir() + '/repos.log'
    serverCfg.logFile = None

    # Transfer SSL settings from rMake config object
    if hasattr(server, 'SSL') and server.SSL:
        # The server supports starting in SSL mode
        serverCfg.useSSL = cfg.reposRequiresSsl()
        serverCfg.sslCert = cfg.sslCertPath
        serverCfg.sslKey = cfg.sslCertPath
    elif cfg.reposRequiresSsl():
        raise errors.RmakeError(
            'Tried to start repository at %s, but missing ssl server library: Please install m2crypto'
            % (cfg.getRepositoryUrl(), ))

    (driver, database) = serverCfg.repositoryDB
    db = dbstore.connect(database, driver)

    # Note - this will automatically migrate this repository!
    # Since this is a throwaway repos anyway, I think that's
    # acceptable.
    compat.ConaryVersion().loadServerSchema(db)
    db.commit()
    db.close()

    user, password = cfg.reposUser.find(cfg.reposName)
    addUser(serverCfg, user, password, write=True)
    if not serverCfg.useSSL:
        # allow anonymous access if we're not securing this repos
        # by using SSL - no use securing access if it's all going to be
        # viewable via tcpdump.
        addUser(serverCfg, 'anonymous', 'anonymous')

    if fork:
        pid = os.fork()
        if pid:
            try:
                pingServer(cfg)
            except:
                killServer(pid)
                raise
            logger.info('Started repository "%s" on port %s (pid %s)' %
                        (cfg.reposName, serverCfg.port, pid))
            return pid
        elif hasattr(logger, 'close'):
            logger.close()
    try:
        os.chdir(cfg.getReposDir())
        serverrc = open(cfg.getReposConfigPath(), 'w')
        serverCfg.store(serverrc, includeDocs=False)
        util.mkdirChain(os.path.dirname(cfg.getReposLogPath()))
        logFile = logfile.LogFile(cfg.getReposLogPath())
        logFile.redirectOutput(close=True)
        serverrc.close()
        os.execv('%s/server/server.py' % conaryDir, [
            '%s/server/server.py' % conaryDir, '--config-file',
            cfg.getReposConfigPath()
        ])
    except Exception, err:
        print >> sys.stderr, 'Could not start repository server: %s' % err
        os._exit(1)
Esempio n. 49
0
    def testVersion20Migration(self):
        dbfile = os.path.join(resources.get_archive(), 'conarydbs',
                              'conarydb-version-19')
        fd, fn = tempfile.mkstemp()
        os.close(fd)
        shutil.copyfile(dbfile, fn)
        # get a list of tables
        db = dbstore.connect(fn, driver='sqlite')
        db.loadSchema()
        cu = db.cursor()
        tableCounts = dict.fromkeys(db.tables.keys())
        for table in tableCounts.keys():
            tableCounts[table] = cu.execute('select count(*) from %s' %
                                            table).fetchall()[0][0]
        # DBInstances is gone...
        tableCounts.pop('DBInstances')
        # we have a VersionId 0 entry now
        tableCounts['Versions'] += 1
        # new table added
        tableCounts['DatabaseAttributes'] = 1

        # do the migration
        db, str = self.captureOutput(sqldb.Database, fn)
        cu = db.db.cursor()

        # make sure we have all the tables
        db2 = dbstore.connect(fn, driver='sqlite')
        db2.loadSchema()
        cu = db2.cursor()
        tableCounts2 = dict.fromkeys(db2.tables.keys())
        for table in tableCounts2.keys():
            tableCounts2[table] = cu.execute('select count(*) from %s' %
                                             table).fetchall()[0][0]
        self.assertEqual(tableCounts, tableCounts2)

        # check to make sure that we fixed our broken deps and troveinfo
        cu.execute("select count(*) from troveinfo where infoType=3 "
                   "and hex(data) == '31'")
        assert (cu.next()[0] == 0)
        cu.execute("select count(*) from dependencies where "
                   "name like 'conary:%' and flag='1'")
        assert (cu.next()[0] == 0)

        # verify the conary:runtime trove
        v = VersionFromString('/conary.rpath.com@rpl:devel//1/1.0-2-0.1')
        f = deps.parseFlavor('~!bootstrap is: x86')
        t = db.getTroves([('conary:runtime', v, f)])[0]
        t.verifyDigitalSignatures()

        # verify that we can insert a '1.0' into deps and troveinfo
        cu.execute("insert into Dependencies values (NULL, 4, 'test', '1.0')")
        cu.execute("select flag from Dependencies where name='test'")
        assert (cu.next()[0] == '1.0')

        cu.execute("insert into TroveInfo values (300, 3, '1.0')")
        cu.execute("select data from TroveInfo where instanceId=300 and "
                   "infotype=3")
        assert (cu.next()[0] == '1.0')

        db.close()
        db2.close()

        # make sure the addition of DatabaseAttributes happens correctly
        db = dbstore.connect(fn, driver='sqlite')
        db.loadSchema()
        self.assertTrue('DatabaseAttributes' in db.tables)
        cu = db.cursor()
        cu.execute("DROP TABLE DatabaseAttributes")
        db.commit()
        db.close()

        sdb = sqldb.Database(fn)
        self.assertTrue('DatabaseAttributes' in sdb.db.tables)
        del sdb

        os.unlink(fn)