class TestRulesSpecial(unittest.TestCase, RulesTestMixin, MemoryDatabaseMixin): def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.createTables() self.rules = self.db.rules self.rules.t.insert().execute(id=1, priority=100, version='4.0*', throttle=100, update_type='z', data_version=1) self.rules.t.insert().execute(id=2, priority=100, channel='release*', throttle=100, update_type='z', data_version=1) def testGetRulesMatchingQueryVersionGlobbing(self): expected = [dict(rule_id=1, priority=100, throttle=100, version='4.0*', update_type='z', data_version=1)] rules = self.rules.getRulesMatchingQuery( dict(name='', product='', version='4.0', channel='', buildTarget='', buildID='', locale='', osVersion='', distribution='', distVersion='', headerArchitecture='' ), fallbackChannel='' ) rules = self._stripNullColumns(rules) self.assertEquals(rules, expected) rules = self.rules.getRulesMatchingQuery( dict(name='', product='', version='4.0b2', channel='', buildTarget='', buildID='', locale='', osVersion='', distribution='', distVersion='', headerArchitecture='' ), fallbackChannel='' ) rules = self._stripNullColumns(rules) self.assertEquals(rules, expected) rules = self.rules.getRulesMatchingQuery( dict(name='', product='', version='4.0.1', channel='', buildTarget='', buildID='', locale='', osVersion='', distribution='', distVersion='', headerArchitecture='' ), fallbackChannel='' ) rules = self._stripNullColumns(rules) self.assertEquals(rules, expected) def testGetRulesMatchingQueryChannelGlobbing(self): expected = [dict(rule_id=2, priority=100, throttle=100, channel='release*', update_type='z', data_version=1)] rules = self.rules.getRulesMatchingQuery( dict(name='', product='', version='', channel='releasetest', buildTarget='', buildID='', locale='', osVersion='', distribution='', distVersion='', headerArchitecture='' ), fallbackChannel='releasetest' ) rules = self._stripNullColumns(rules) self.assertEquals(rules, expected) rules = self.rules.getRulesMatchingQuery( dict(name='', product='', version='', channel='releasetest-cck-blah', buildTarget='', buildID='', locale='', osVersion='', distribution='', distVersion='', headerArchitecture='' ), fallbackChannel='releasetest' ) rules = self._stripNullColumns(rules) self.assertEquals(rules, expected)
class TestReleases(unittest.TestCase, MemoryDatabaseMixin): def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.create() self.releases = self.db.releases self.releases.t.insert().execute(name='a', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1) self.releases.t.insert().execute(name='ab', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1) self.releases.t.insert().execute(name='b', product='b', version='b', data=json.dumps(dict(name=2)), data_version=1) self.releases.t.insert().execute(name='c', product='c', version='c', data=json.dumps(dict(name=3)), data_version=1) def testGetReleases(self): self.assertEquals(len(self.releases.getReleases()), 4) def testGetReleasesWithLimit(self): self.assertEquals(len(self.releases.getReleases(limit=1)), 1) def testGetReleasesWithWhere(self): expected = [dict(product='b', version='b', name='b', data=dict(name=2), data_version=1)] self.assertEquals(self.releases.getReleases(name='b'), expected) def testGetReleaseBlob(self): expected = dict(name=3) self.assertEquals(self.releases.getReleaseBlob(name='c'), expected) def testGetReleaseBlobNonExistentRelease(self): self.assertRaises(KeyError, self.releases.getReleaseBlob, name='z') def testGetReleaseNames(self): releases = self.releases.getReleaseNames() expected = [ dict(name='a'), dict(name='ab'), dict(name='b'), dict(name='c')] self.assertEquals(releases, expected) releases = self.releases.getReleaseNames(product='a') expected = [ dict(name='a'), dict(name='ab')] self.assertEquals(releases, expected) releases = self.releases.getReleaseNames(version='b') expected = [ dict(name='b'), ] self.assertEquals(releases, expected) releases = self.releases.getReleaseNames(product='a', version='b') expected = [ ] self.assertEquals(releases, expected)
def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.createTables() self.rules = self.db.rules self.rules.t.insert().execute(id=1, priority=100, version='4.0*', throttle=100, update_type='z', data_version=1) self.rules.t.insert().execute(id=2, priority=100, channel='release*', throttle=100, update_type='z', data_version=1)
def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.createTables() self.releases = self.db.releases self.releases.t.insert().execute(name='a', product='a', version='a', data_version=1, data=""" { "name": "a", "platforms": { "p": { "locales": { "l": { "complete": { "filesize": "1234" } } } } } } """) self.releases.t.insert().execute(name='b', product='b', version='b', data_version=1, data=""" { "name": "b" } """)
def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.createTables() self.releases = self.db.releases self.releases.t.insert().execute(name='a', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1) self.releases.t.insert().execute(name='ab', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1) self.releases.t.insert().execute(name='b', product='b', version='b', data=json.dumps(dict(name=2)), data_version=1) self.releases.t.insert().execute(name='c', product='c', version='c', data=json.dumps(dict(name=3)), data_version=1)
def testReset(self): db = AUSDatabase('sqlite:///:memory:') db.reset() # If we can set the dburi again, reset worked! db.setDburi('sqlite:///:memory:') db.createTables() insp = Inspector.from_engine(db.engine) self.assertNotEqual(insp.get_table_names(), [])
def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.createTables() self.paths = self.db.rules self.paths.t.insert().execute(id=1, priority=100, version='3.5', buildTarget='d', throttle=100, mapping='c', update_type='z', data_version=1) self.paths.t.insert().execute(id=2, priority=100, version='3.3', buildTarget='d', throttle=100, mapping='b', update_type='z', data_version=1) self.paths.t.insert().execute(id=3, priority=100, version='3.5', buildTarget='a', throttle=100, mapping='a', update_type='z', data_version=1) self.paths.t.insert().execute(id=4, priority=80, buildTarget='d', throttle=100, mapping='a', update_type='z', data_version=1) self.paths.t.insert().execute(id=5, priority=80, buildTarget='d', version='3.3', throttle=0, mapping='c', update_type='z', data_version=1)
def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.createTables() self.permissions = self.db.permissions self.permissions.t.insert().execute(permission='admin', username='******', data_version=1) self.permissions.t.insert().execute(permission='/users/:id/permissions/:permission', username='******', data_version=1) self.permissions.t.insert().execute(permission='/releases/:name', username='******', options=json.dumps(dict(product='fake')), data_version=1) self.permissions.t.insert().execute(permission='/rules', username='******', data_version=1) self.permissions.t.insert().execute(permission='/rules/:id', username='******', options=json.dumps(dict(method='POST')), data_version=1)
class TestReleases(unittest.TestCase, MemoryDatabaseMixin): def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.createTables() self.releases = self.db.releases self.releases.t.insert().execute(name='a', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1) self.releases.t.insert().execute(name='ab', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1) self.releases.t.insert().execute(name='b', product='b', version='b', data=json.dumps(dict(name=2)), data_version=1) self.releases.t.insert().execute(name='c', product='c', version='c', data=json.dumps(dict(name=3)), data_version=1) def testGetReleases(self): self.assertEquals(len(self.releases.getReleases()), 4) def testGetReleasesWithLimit(self): self.assertEquals(len(self.releases.getReleases(limit=1)), 1) def testGetReleasesWithWhere(self): expected = [dict(product='b', version='b', name='b', data=dict(name=2), data_version=1)] self.assertEquals(self.releases.getReleases(name='b'), expected) def testGetReleaseBlob(self): expected = dict(name=3) self.assertEquals(self.releases.getReleaseBlob(name='c'), expected) def testGetReleaseBlobNonExistentRelease(self): self.assertRaises(KeyError, self.releases.getReleaseBlob, name='z') def testAddRelease(self): blob = ReleaseBlobV1(name=4) self.releases.addRelease(name='d', product='d', version='d', blob=blob, changed_by='bill') expected = [('d', 'd', 'd', json.dumps(dict(name=4)), 1)] self.assertEquals(self.releases.t.select().where(self.releases.name=='d').execute().fetchall(), expected) def testAddReleaseAlreadyExists(self): blob = ReleaseBlobV1(name=1) self.assertRaises(TransactionError, self.releases.addRelease, name='a', product='a', version='a', blob=blob, changed_by='bill') def testUpdateRelease(self): self.releases.updateRelease(name='b', product='z', version='y', changed_by='bill', old_data_version=1) expected = [('b', 'z', 'y', json.dumps(dict(name=2)), 2)] self.assertEquals(self.releases.t.select().where(self.releases.name=='b').execute().fetchall(), expected)
def setDb(self, dburi, releases_history_buckets=None, releases_history_class=None): from auslib.db import AUSDatabase, GCSHistory if not releases_history_class and releases_history_buckets is not None: releases_history_class = GCSHistory self.db = AUSDatabase( dburi, releases_history_buckets=releases_history_buckets, releases_history_class=releases_history_class)
from argparse import ArgumentParser parser = ArgumentParser() parser.add_argument("--db", dest="db", help="The database to use, in URI format. Incompatible with --ini.") parser.add_argument("--ini", dest="ini", help="The config file to look for the database in. Incompatible with --db.") parser.add_argument("--name", dest="name", required=True, help="The name of the person/script doing the migrations.") parser.add_argument("releases", metavar="release", nargs="+", help="The releases (aka blob names) to migrate.") args = parser.parse_args() if args.db and args.ini: parser.error("Cannot specify --db and --ini!") if args.ini: cfg = AdminConfig(args.ini) db = AUSDatabase(cfg.getDburi()) db.setDomainWhitelist(cfg.getDomainWhitelist()) else: db = AUSDatabase(args.db) for release in args.releases: try: rel = db.releases.getReleases(name=release)[0] blob = rel["data"] except: log.debug("No such release '%s', skipping", release) if blob["schema_version"] == 4: log.debug("%s is already schema 4, skipping", release) continue elif blob["schema_version"] != 3:
def setDb(self, dburi): from auslib.db import AUSDatabase self.db = AUSDatabase(dburi)
"Anything older than this will be deleted.\n" ) usage += " cleanup-dryrun: Show what would be removed if 'cleanup' is run." parser = OptionParser(usage=usage) parser.add_option("-d", "--db", dest="db", default=None, help="database to manage, in URI format") parser.add_option("--version", dest="version", default=None, type="int", help="Create/upgrade to this specific schema version rather than the latest.") options, args = parser.parse_args() if not options.db: parser.error("db is required") if len(args) < 1: parser.error("need an action to perform") action = args[0] db = AUSDatabase(options.db, mysql_traditional_mode=True) if action == "create": db.create(options.version) elif action == "upgrade": db.upgrade(options.version) elif action == "downgrade": db.downgrade(options.version) elif action == "extract": with db.begin() as trans: if len(args) < 2: extract_active_data(trans, options.db) else: location = args[1] extract_active_data(trans, options.db, location) elif action.startswith("cleanup"): if len(args) < 2:
if __name__ == "__main__": from optparse import OptionParser usage = """%s --db dburi action\n""" % sys.argv[0] usage += "Possible actions:\n" usage += " create: Create all the tables required for a new Balrog database\n" usage += " upgrade: Upgrade an existing balrog table to a newer version." parser = OptionParser(usage=usage) parser.add_option("-d", "--db", dest="db", default=None, help="database to manage, in URI format") parser.add_option( "--version", dest="version", default=None, type="int", help="Create/upgrade to this specific schema version rather than the latest.", ) options, args = parser.parse_args() if not options.db: parser.error("db is required") if len(args) != 1: parser.error("need a single action to perform") action = args[0] db = AUSDatabase(options.db) if action == "create": db.create(options.version) elif action == "upgrade": db.upgrade(options.version)
dest="name", required=True, help="The name of the person/script doing the migrations.") parser.add_argument("releases", metavar="release", nargs="+", help="The releases (aka blob names) to migrate.") args = parser.parse_args() if args.db and args.ini: parser.error("Cannot specify --db and --ini!") if args.ini: cfg = AdminConfig(args.ini) db = AUSDatabase(cfg.getDburi()) db.setDomainWhitelist(cfg.getDomainWhitelist()) else: db = AUSDatabase(args.db) for release in args.releases: try: rel = db.releases.getReleases(name=release)[0] blob = rel["data"] except: log.debug("No such release '%s', skipping", release) if blob["schema_version"] == 4: log.debug("%s is already schema 4, skipping", release) continue elif blob["schema_version"] != 3:
class TestReleasesSchema1(unittest.TestCase, MemoryDatabaseMixin): """Tests for the Releases class that depend on version 1 of the blob schema.""" def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.create() self.releases = self.db.releases self.releases.t.insert().execute(name='a', product='a', version='a', data_version=1, data=""" { "name": "a", "platforms": { "p": { "locales": { "l": { "complete": { "filesize": 1234 } } } }, "p2": { "alias": "p" } } } """) self.releases.t.insert().execute(name='b', product='b', version='b', data_version=1, data=""" { "name": "b" } """) def testAddRelease(self): blob = ReleaseBlobV1(name=4) self.releases.addRelease(name='d', product='d', version='d', blob=blob, changed_by='bill') expected = [('d', 'd', 'd', json.dumps(dict(name=4)), 1)] self.assertEquals(self.releases.t.select().where(self.releases.name=='d').execute().fetchall(), expected) def testAddReleaseAlreadyExists(self): blob = ReleaseBlobV1(name=1) self.assertRaises(TransactionError, self.releases.addRelease, name='a', product='a', version='a', blob=blob, changed_by='bill') def testUpdateRelease(self): blob = ReleaseBlobV1(name='a') self.releases.updateRelease(name='b', product='z', version='y', blob=blob, changed_by='bill', old_data_version=1) expected = [('b', 'z', 'y', json.dumps(dict(name='a')), 2)] self.assertEquals(self.releases.t.select().where(self.releases.name=='b').execute().fetchall(), expected) def testUpdateReleaseWithBlob(self): blob = ReleaseBlobV1(name='b', schema_version=3) self.releases.updateRelease(name='b', product='z', version='y', changed_by='bill', blob=blob, old_data_version=1) expected = [('b', 'z', 'y', json.dumps(dict(name='b', schema_version=3)), 2)] self.assertEquals(self.releases.t.select().where(self.releases.name=='b').execute().fetchall(), expected) def testUpdateReleaseInvalidBlob(self): blob = ReleaseBlobV1(name=2) blob['foo'] = 'bar' self.assertRaises(ValueError, self.releases.updateRelease, changed_by='bill', name='b', blob=blob, old_data_version=1) def testAddLocaleToRelease(self): data = dict(complete=dict(hashValue='abc')) self.releases.addLocaleToRelease(name='a', platform='p', locale='c', data=data, old_data_version=1, changed_by='bill') ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0]) expected = json.loads(""" { "name": "a", "platforms": { "p": { "locales": { "c": { "complete": { "hashValue": "abc" } }, "l": { "complete": { "filesize": 1234 } } } }, "p2": { "alias": "p" } } } """) self.assertEqual(ret, expected) def testAddLocaleToReleaseOverride(self): data = dict(complete=dict(hashValue=789)) self.releases.addLocaleToRelease(name='a', platform='p', locale='l', data=data, old_data_version=1, changed_by='bill') ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0]) expected = json.loads(""" { "name": "a", "platforms": { "p": { "locales": { "l": { "complete": { "hashValue": 789 } } } }, "p2": { "alias": "p" } } } """) self.assertEqual(ret, expected) def testAddLocaleToReleasePlatformsDoesntExist(self): data = dict(complete=dict(filesize=432)) self.releases.addLocaleToRelease(name='b', platform='q', locale='l', data=data, old_data_version=1, changed_by='bill') ret = json.loads(select([self.releases.data]).where(self.releases.name=='b').execute().fetchone()[0]) expected = json.loads(""" { "name": "b", "platforms": { "q": { "locales": { "l": { "complete": { "filesize": 432 } } } } } } """) self.assertEqual(ret, expected) def testAddLocaleToReleaseSecondPlatform(self): data = dict(complete=dict(filesize=324)) self.releases.addLocaleToRelease(name='a', platform='q', locale='l', data=data, old_data_version=1, changed_by='bill') ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0]) expected = json.loads(""" { "name": "a", "platforms": { "p": { "locales": { "l": { "complete": { "filesize": 1234 } } } }, "p2": { "alias": "p" }, "q": { "locales": { "l": { "complete": { "filesize": 324 } } } } } } """) self.assertEqual(ret, expected) def testAddLocaleToReleaseResolveAlias(self): data = dict(complete=dict(filesize=444)) self.releases.addLocaleToRelease(name='a', platform='p2', locale='j', data=data, old_data_version=1, changed_by='bill') ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0]) expected = json.loads(""" { "name": "a", "platforms": { "p": { "locales": { "l": { "complete": { "filesize": 1234 } }, "j": { "complete": { "filesize": 444 } } } }, "p2": { "alias": "p" } } } """) self.assertEqual(ret, expected)
def testCreateTables(self): db = AUSDatabase() db.setDburi('sqlite:///:memory:') db.createTables() insp = Inspector.from_engine(db.engine) self.assertNotEqual(insp.get_table_names(), [])
class TestRulesSimple(unittest.TestCase, RulesTestMixin, MemoryDatabaseMixin): def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.create() self.paths = self.db.rules self.paths.t.insert().execute(id=1, priority=100, version='3.5', buildTarget='d', throttle=100, mapping='c', update_type='z', data_version=1) self.paths.t.insert().execute(id=2, priority=100, version='3.3', buildTarget='d', throttle=100, mapping='b', update_type='z', data_version=1) self.paths.t.insert().execute(id=3, priority=100, version='3.5', buildTarget='a', throttle=100, mapping='a', update_type='z', data_version=1) self.paths.t.insert().execute(id=4, priority=80, buildTarget='d', throttle=100, mapping='a', update_type='z', data_version=1) self.paths.t.insert().execute(id=5, priority=80, buildTarget='d', version='3.3', throttle=0, mapping='c', update_type='z', data_version=1) def testGetOrderedRules(self): rules = self._stripNullColumns(self.paths.getOrderedRules()) expected = [ dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1), dict(rule_id=5, priority=80, throttle=0, version='3.3', buildTarget='d', mapping='c', update_type='z', data_version=1), dict(rule_id=2, priority=100, throttle=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1), dict(rule_id=3, priority=100, throttle=100, version='3.5', buildTarget='a', mapping='a', update_type='z', data_version=1), dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1), ] self.assertEquals(rules, expected) def testGetRulesMatchingQuery(self): rules = self.paths.getRulesMatchingQuery( dict(name='', product='', version='3.5', channel='', buildTarget='a', buildID='', locale='', osVersion='', distribution='', distVersion='', headerArchitecture='', force=False ), fallbackChannel='' ) rules = self._stripNullColumns(rules) expected = [dict(rule_id=3, priority=100, throttle=100, version='3.5', buildTarget='a', mapping='a', update_type='z', data_version=1)] self.assertEquals(rules, expected) def testGetRulesMatchingQueryWithNullColumn(self): rules = self.paths.getRulesMatchingQuery( dict(name='', product='', version='3.5', channel='', buildTarget='d', buildID='', locale='', osVersion='', distribution='', distVersion='', headerArchitecture='', force=False ), fallbackChannel='' ) rules = self._stripNullColumns(rules) expected = [ dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1), dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1), ] self.assertEquals(rules, expected) def testGetRulesMatchingQueryDontReturnThrottled(self): rules = self.paths.getRulesMatchingQuery( dict(name='', product='', version='3.3', channel='', buildTarget='d', buildID='', locale='', osVersion='', distribution='', distVersion='', headerArchitecture='', force=False ), fallbackChannel='' ) rules = self._stripNullColumns(rules) expected = [ dict(rule_id=2, priority=100, throttle=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1), dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1), ] self.assertEquals(rules, expected) def testGetRulesMatchingQueryReturnThrottled(self): rules = self.paths.getRulesMatchingQuery( dict(name='', product='', version='3.3', channel='', buildTarget='d', buildID='', locale='', osVersion='', distribution='', distVersion='', headerArchitecture='', force=True ), fallbackChannel='' ) rules = self._stripNullColumns(rules) expected = [ dict(rule_id=2, priority=100, throttle=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1), dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1), dict(rule_id=5, priority=80, throttle=0, version='3.3', buildTarget='d', mapping='c', update_type='z', data_version=1) ] self.assertEquals(rules, expected) def testGetRuleById(self): rule = self._stripNullColumns([self.paths.getRuleById(1)]) expected = [dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1)] self.assertEquals(rule, expected) def testAddRule(self): what = dict(throttle=11, mapping='c', update_type='z', priority=60) rule_id = self.paths.addRule(changed_by='bill', what=what) rule_id = rule_id[0] rules = self.paths.t.select().where(self.paths.rule_id==rule_id).execute().fetchall() copy_rule = dict(rules[0].items()) rule = self._stripNullColumns( [copy_rule] ) what['rule_id']=rule_id what['data_version']=1 what = [what] self.assertEquals(rule, what) def testUpdateRule(self): rules = self.paths.t.select().where(self.paths.rule_id==1).execute().fetchall() what = dict(rules[0].items()) what['mapping'] = 'd' self.paths.updateRule(changed_by='bill', rule_id=1, what=what, old_data_version=1) rules = self.paths.t.select().where(self.paths.rule_id==1).execute().fetchall() copy_rule = dict(rules[0].items()) rule = self._stripNullColumns( [copy_rule] ) expected = [dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='d', update_type='z', data_version=1)] self.assertEquals(rule, expected)
class AUS3: def __init__(self, dbname=None): self.setDb(dbname) def setDb(self, dbname): if dbname == None: dbname = "sqlite:///update.db" self.db = AUSDatabase(dbname) self.releases = self.db.releases self.rules = self.db.rules def createTables(self): self.db.createTables() def identifyRequest(self, updateQuery): log.debug("AUS.identifyRequest: got updateQuery: %s", updateQuery) buildTarget = updateQuery["buildTarget"] buildID = updateQuery["buildID"] locale = updateQuery["locale"] for release in self.releases.getReleases(product=updateQuery["product"], version=updateQuery["version"]): log.debug("AUS.identifyRequest: Trying to match request to %s", release["name"]) if buildTarget in release["data"]["platforms"]: releaseBuildID = release["data"].getBuildID(buildTarget, locale) log.debug("AUS.identifyRequest: releasePlat buildID is: %s", releaseBuildID) if buildID == releaseBuildID: log.debug("AUS.identifyRequest: Identified query as %s", release["name"]) return release["name"] log.debug("AUS.identifyRequest: Couldn't identify query") return None def evaluateRules(self, updateQuery): log.debug("AUS.evaluateRules: Looking for rules that apply to:") log.debug("AUS.evaluateRules: %s", updateQuery) rules = self.rules.getRulesMatchingQuery( updateQuery, fallbackChannel=self.getFallbackChannel(updateQuery["channel"]) ) ### XXX throw any N->N update rules and keep the highest priority remaining one if len(rules) >= 1: rules = sorted(rules, key=lambda rule: rule["priority"], reverse=True) log.debug("AUS.evaluateRules: Returning rule:") log.debug("AUS.evaluateRules: %s", rules[0]) return rules[0] return None def getFallbackChannel(self, channel): return channel.split("-cck-")[0] def expandRelease(self, updateQuery, rule): if not rule or not rule["mapping"]: log.debug("AUS.expandRelease: Couldn't find rule or mapping for %s" % rule) return None # read data from releases table try: res = self.releases.getReleases(name=rule["mapping"], limit=1)[0] except IndexError: # need to log some sort of data inconsistency error here log.debug("AUS.expandRelease: Failed to get release data from db for:") log.debug("AUS.expandRelease: %s", rule["mapping"]) return None relData = res["data"] updateData = defaultdict(list) # platforms may be aliased to another platform in the case # of identical data, minimizing the json size buildTarget = updateQuery["buildTarget"] relDataPlat = relData.getPlatformData(buildTarget) locale = updateQuery["locale"] # return early if we don't have an update for this platform if buildTarget not in relData["platforms"]: log.debug("AUS.expandRelease: No platform %s in release %s", buildTarget, rule["mapping"]) return updateData # return early if we don't have an update for this locale if locale not in relDataPlat["locales"]: log.debug("AUS.expandRelease: No update to %s for %s/%s", rule["mapping"], buildTarget, locale) return updateData else: relDataPlatLoc = relDataPlat["locales"][locale] # this is for the properties AUS2 can cope with today if relData["schema_version"] == 1: updateData["type"] = rule["update_type"] updateData["appv"] = relData.getAppv(buildTarget, locale) updateData["extv"] = relData.getExtv(buildTarget, locale) updateData["schema_version"] = relData["schema_version"] if "detailsUrl" in relData: updateData["detailsUrl"] = relData["detailsUrl"].replace("%LOCALE%", updateQuery["locale"]) updateData["build"] = relData.getBuildID(buildTarget, locale) # evaluate types of updates and see if we can use them for patchKey in relDataPlatLoc: if patchKey not in ("partial", "complete"): log.debug("AUS.expandRelease: Skipping patchKey '%s'", patchKey) continue patch = relDataPlatLoc[patchKey] if patch["from"] == updateQuery["name"] or patch["from"] == "*": if "fileUrl" in patch: url = patch["fileUrl"] else: # When we're using a fallback channel it's unlikely # we'll have a fileUrl specifically for it, but we # should try nonetheless. Non-fallback cases shouldn't # be hitting any exceptions here. try: url = relData["fileUrls"][updateQuery["channel"]] except KeyError: url = relData["fileUrls"][self.getFallbackChannel(updateQuery["channel"])] url = url.replace("%LOCALE%", updateQuery["locale"]) url = url.replace("%OS_FTP%", relDataPlat["OS_FTP"]) url = url.replace("%FILENAME%", relData["ftpFilenames"][patchKey]) url = url.replace("%PRODUCT%", relData["bouncerProducts"][patchKey]) url = url.replace("%OS_BOUNCER%", relDataPlat["OS_BOUNCER"]) updateData["patches"].append( { "type": patchKey, "URL": url, "hashFunction": relData["hashFunction"], "hashValue": patch["hashValue"], "size": patch["filesize"], } ) else: log.debug( "AUS.expandRelease: Didn't add patch for patchKey '%s'; from is '%s', updateQuery name is '%s'", patchKey, patch["from"], updateQuery["name"], ) # older branches required a <partial> in the update.xml, which we # used to fake by repeating the complete data. if ( "fakePartials" in relData and relData["fakePartials"] and len(updateData["patches"]) == 1 and updateData["patches"][0]["type"] == "complete" ): patch = copy.copy(updateData["patches"][0]) patch["type"] = "partial" updateData["patches"].append(patch) log.debug("AUS.expandRelease: Returning %s", updateData) return updateData def createSnippet(self, updateQuery, release): rel = self.expandRelease(updateQuery, release) if not rel: # handle this better, both for prod and debugging log.debug("AUS.createSnippet: Couldn't expand rule for update target") # XXX: Not sure we should be specifying patch types here, but it's # required for tests that have null snippets in them at the time # of writing. return {"partial": "", "complete": ""} snippets = {} for patch in rel["patches"]: snippet = [ "version=1", "type=%s" % patch["type"], "url=%s" % patch["URL"], "hashFunction=%s" % patch["hashFunction"], "hashValue=%s" % patch["hashValue"], "size=%s" % patch["size"], "build=%s" % rel["build"], "appv=%s" % rel["appv"], "extv=%s" % rel["extv"], ] if rel["detailsUrl"]: snippet.append("detailsUrl=%s" % rel["detailsUrl"]) if rel["type"] == "major": snippets.append("updateType=major") # AUS2 snippets have a trailing newline, add one here for easy diffing snippets[patch["type"]] = "\n".join(snippet) + "\n" # XXX: need to handle old releases needing completes duplicating partials # add another parameter in the rule table and use it here return snippets def createXML(self, updateQuery, release): rel = self.expandRelease(updateQuery, release) # this will fall down all sorts of interesting ways by hardcoding fields xml = ['<?xml version="1.0"?>'] xml.append("<updates>") if rel: if rel["schema_version"] == 1: xml.append( ' <update type="%s" version="%s" extensionVersion="%s" buildID="%s"' % (rel["type"], rel["appv"], rel["extv"], rel["build"]) ) if rel["detailsUrl"]: xml.append(' detailsURL="%s"' % rel["detailsUrl"]) xml.append(">") for patch in sorted(rel["patches"]): xml.append( ' <patch type="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s"/>' % (patch["type"], patch["URL"], patch["hashFunction"], patch["hashValue"], patch["size"]) ) # XXX: need to handle old releases needing completes duplicating partials # add another parameter in the rule table and use it here xml.append(" </update>") xml.append("</updates>") return "\n".join(xml)
sys.path.append(path.join(path.dirname(__file__), "..")) from auslib.blobs.apprelease import ReleaseBlobV1 from auslib.db import AUSDatabase if __name__ == "__main__": from optparse import OptionParser doc = "%s --db dburi -r release-name -v version -p product foo.json" % sys.argv[0] parser = OptionParser(doc) parser.add_option("-d", "--db", dest="db", default=None, help="database to manage, in URI format") parser.add_option("-r", "--release", dest="release", default=None, help="Release to put blob into") parser.add_option("-v", "--version", dest="version", default=None, help="Version of the release") parser.add_option("-p", "--product", dest="product", default=None, help="Product of the release") options, args = parser.parse_args() logging.basicConfig(level=logging.DEBUG, format="%(asctime)s: %(message)s") if not options.db or not options.release or not options.version or not options.product or len(args) != 1: print doc sys.exit(1) db = AUSDatabase(options.db) blob = ReleaseBlobV1() blob.loadJSON(open(args[0]).read()) try: old = db.releases.getReleases(name=options.release)[0] db.releases.updateRelease(name=options.release, product=options.product, version=options.version, changed_by='import-json', old_data_version=old['data_version'], blob=blob) except IndexError: db.releases.addRelease(name=options.release, product=options.product, version=options.version, blob=blob, changed_by='import-json')
class TestReleasesSchema1(unittest.TestCase, MemoryDatabaseMixin): """Tests for the Releases class that depend on version 1 of the blob schema.""" def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.createTables() self.releases = self.db.releases self.releases.t.insert().execute(name='a', product='a', version='a', data_version=1, data=""" { "name": "a", "platforms": { "p": { "locales": { "l": { "complete": { "filesize": "1234" } } } } } } """) self.releases.t.insert().execute(name='b', product='b', version='b', data_version=1, data=""" { "name": "b" } """) def testAddLocaleToRelease(self): blob = dict(complete=dict(hashValue='abc')) self.releases.addLocaleToRelease(name='a', platform='p', locale='c', blob=blob, old_data_version=1, changed_by='bill') ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0]) expected = json.loads(""" { "name": "a", "platforms": { "p": { "locales": { "c": { "complete": { "hashValue": "abc" } }, "l": { "complete": { "filesize": "1234" } } } } } } """) self.assertEqual(ret, expected) def testAddLocaleToReleaseOverride(self): blob = dict(complete=dict(hashValue=789)) self.releases.addLocaleToRelease(name='a', platform='p', locale='l', blob=blob, old_data_version=1, changed_by='bill') ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0]) expected = json.loads(""" { "name": "a", "platforms": { "p": { "locales": { "l": { "complete": { "hashValue": 789 } } } } } } """) self.assertEqual(ret, expected) def testAddLocaleToReleasePlatformsDoesntExist(self): blob = dict(complete=dict(filesize=432)) self.releases.addLocaleToRelease(name='b', platform='q', locale='l', blob=blob, old_data_version=1, changed_by='bill') ret = json.loads(select([self.releases.data]).where(self.releases.name=='b').execute().fetchone()[0]) expected = json.loads(""" { "name": "b", "platforms": { "q": { "locales": { "l": { "complete": { "filesize": 432 } } } } } } """) self.assertEqual(ret, expected)
class TestRulesSimple(unittest.TestCase, RulesTestMixin, MemoryDatabaseMixin): def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.createTables() self.paths = self.db.rules self.paths.t.insert().execute(id=1, priority=100, version='3.5', buildTarget='d', throttle=100, mapping='c', update_type='z', data_version=1) self.paths.t.insert().execute(id=2, priority=100, version='3.3', buildTarget='d', throttle=100, mapping='b', update_type='z', data_version=1) self.paths.t.insert().execute(id=3, priority=100, version='3.5', buildTarget='a', throttle=100, mapping='a', update_type='z', data_version=1) self.paths.t.insert().execute(id=4, priority=80, buildTarget='d', throttle=100, mapping='a', update_type='z', data_version=1) self.paths.t.insert().execute(id=5, priority=80, buildTarget='d', version='3.3', throttle=0, mapping='c', update_type='z', data_version=1) def testGetOrderedRules(self): rules = self._stripNullColumns(self.paths.getOrderedRules()) expected = [ dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1), dict(rule_id=5, priority=80, throttle=0, version='3.3', buildTarget='d', mapping='c', update_type='z', data_version=1), dict(rule_id=2, priority=100, throttle=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1), dict(rule_id=3, priority=100, throttle=100, version='3.5', buildTarget='a', mapping='a', update_type='z', data_version=1), dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1), ] self.assertEquals(rules, expected) def testGetRulesMatchingQuery(self): rules = self.paths.getRulesMatchingQuery( dict(name='', product='', version='3.5', channel='', buildTarget='a', buildID='', locale='', osVersion='', distribution='', distVersion='', headerArchitecture='' ), fallbackChannel='' ) rules = self._stripNullColumns(rules) expected = [dict(rule_id=3, priority=100, throttle=100, version='3.5', buildTarget='a', mapping='a', update_type='z', data_version=1)] self.assertEquals(rules, expected) def testGetRulesMatchingQueryWithNullColumn(self): rules = self.paths.getRulesMatchingQuery( dict(name='', product='', version='3.5', channel='', buildTarget='d', buildID='', locale='', osVersion='', distribution='', distVersion='', headerArchitecture='' ), fallbackChannel='' ) rules = self._stripNullColumns(rules) expected = [ dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1), dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1), ] self.assertEquals(rules, expected) def testGetRulesMatchingQueryDontReturnThrottled(self): rules = self.paths.getRulesMatchingQuery( dict(name='', product='', version='3.3', channel='', buildTarget='d', buildID='', locale='', osVersion='', distribution='', distVersion='', headerArchitecture='' ), fallbackChannel='' ) rules = self._stripNullColumns(rules) expected = [ dict(rule_id=2, priority=100, throttle=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1), dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1), ] self.assertEquals(rules, expected)
usage += " cleanup: Cleanup old data from a database. Requires an extra arg of maximum age (in days) of nightly releases. " \ "Anything older than this will be deleted.\n" usage += " cleanup-dryrun: Show what would be removed if 'cleanup' is run." parser = OptionParser(usage=usage) parser.add_option("-d", "--db", dest="db", default=None, help="database to manage, in URI format") parser.add_option("--version", dest="version", default=None, type="int", help="Create/upgrade to this specific schema version rather than the latest.") options, args = parser.parse_args() if not options.db: parser.error("db is required") if len(args) < 1: parser.error("need an action to perform") action = args[0] db = AUSDatabase(options.db, mysql_traditional_mode=True) if action == 'create': db.create(options.version) elif action == 'upgrade': db.upgrade(options.version) elif action == 'downgrade': db.downgrade(options.version) elif action.startswith("cleanup"): if len(args) < 2: parser.error("need to pass maximum nightly release age") nightly_age = int(args[1]) with db.begin() as trans: if action == "cleanup": cleanup_releases(trans, nightly_age, dryrun=False) cleanup_releases_history(trans, dryrun=False) else:
class TestPermissions(unittest.TestCase, MemoryDatabaseMixin): def setUp(self): MemoryDatabaseMixin.setUp(self) self.db = AUSDatabase(self.dburi) self.db.createTables() self.permissions = self.db.permissions self.permissions.t.insert().execute(permission='admin', username='******', data_version=1) self.permissions.t.insert().execute(permission='/users/:id/permissions/:permission', username='******', data_version=1) self.permissions.t.insert().execute(permission='/releases/:name', username='******', options=json.dumps(dict(product='fake')), data_version=1) self.permissions.t.insert().execute(permission='/rules', username='******', data_version=1) self.permissions.t.insert().execute(permission='/rules/:id', username='******', options=json.dumps(dict(method='POST')), data_version=1) def testGrantPermissions(self): query = self.permissions.t.select().where(self.permissions.username=='jess') self.assertEquals(len(query.execute().fetchall()), 0) self.permissions.grantPermission('bob', 'jess', '/rules/:id') self.assertEquals(query.execute().fetchall(), [('/rules/:id', 'jess', None, 1)]) def testGrantPermissionsWithOptions(self): self.permissions.grantPermission('bob', 'cathy', '/releases/:name', options=dict(product='SeaMonkey')) query = self.permissions.t.select().where(self.permissions.username=='cathy') query = query.where(self.permissions.permission=='/releases/:name') self.assertEquals(query.execute().fetchall(), [('/releases/:name', 'cathy', json.dumps(dict(product='SeaMonkey')), 1)]) def testGrantPermissionsNotAllowed(self): self.assertRaises(PermissionDeniedError, self.permissions.grantPermission, 'cathy', 'bob', '/rules/:id' ) def testGrantPermissionsUnknownPermission(self): self.assertRaises(ValueError, self.permissions.grantPermission, 'bob', 'bud', 'bad' ) def testGrantPermissionsUnknownOption(self): self.assertRaises(ValueError, self.permissions.grantPermission, 'bob', 'bud', '/rules/:id', dict(foo=1) ) def testRevokePermission(self): self.permissions.revokePermission(changed_by='bill', username='******', permission='/releases/:name', old_data_version=1) query = self.permissions.t.select().where(self.permissions.username=='bob') query = query.where(self.permissions.permission=='/releases/:name') self.assertEquals(len(query.execute().fetchall()), 0) def testCanEditUsers(self): self.assertTrue(self.permissions.canEditUsers('bill')) self.assertTrue(self.permissions.canEditUsers('bob')) def testCanEditUsersFalse(self): self.assertFalse(self.permissions.canEditUsers('cathy')) def testGetAllUsers(self): self.assertEquals(self.permissions.getAllUsers(), ['bill', 'bob', 'cathy']) def testGetUserPermissions(self): expected = {'/users/:id/permissions/:permission': dict(options=None, data_version=1), '/releases/:name': dict(options=dict(product='fake'), data_version=1), '/rules/:id': dict(options=dict(method='POST'), data_version=1)} self.assertEquals(self.permissions.getUserPermissions('bob'), expected) def testGetOptions(self): expected = dict(product='fake') self.assertEquals(self.permissions.getOptions('bob', '/releases/:name'), expected) def testGetOptionsPermissionDoesntExist(self): self.assertRaises(ValueError, self.permissions.getOptions, 'fake', 'fake') def testGetOptionsNoOptions(self): self.assertEquals(self.permissions.getOptions('cathy', '/rules'), {}) def testHasUrlPermission(self): self.assertTrue(self.permissions.hasUrlPermission('cathy', '/rules', 'PUT', dict(product='fake'))) def testHasUrlPermissionWithOption(self): self.assertTrue(self.permissions.hasUrlPermission('bob', '/rules/:id', 'POST', dict(product='fake'))) def testHasUrlPermissionNotAllowed(self): self.assertFalse(self.permissions.hasUrlPermission('cathy', '/rules/:id', 'DELETE', dict(product='fake'))) def testHasUrlPermissionNotAllowedWithOption(self): self.assertFalse(self.permissions.hasUrlPermission('bob', '/rules/:id', 'DELETE', dict(product='fake'))) def testHasUrlPermissionWithProduct(self): self.assertTrue(self.permissions.hasUrlPermission('bob', '/releases/:name', 'DELETE', dict(product='fake')))
def setDb(self, dbname): if dbname == None: dbname = "sqlite:///update.db" self.db = AUSDatabase(dbname) self.releases = self.db.releases self.rules = self.db.rules
def testModelIsSameAsRepository(self): db2 = AUSDatabase('sqlite:///' + self.getTempfile()) db2.create() diff = migrate.versioning.api.compare_model_to_db(db2.engine, self.db.migrate_repo, self.db.metadata) if diff: self.fail(str(diff))