class TestRulesSpecial(unittest.TestCase, RulesTestMixin, MemoryDatabaseMixin):
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.rules = self.db.rules
self.rules.t.insert().execute(id=1, priority=100, version='4.0*', throttle=100, update_type='z', data_version=1)
self.rules.t.insert().execute(id=2, priority=100, channel='release*', throttle=100, update_type='z', data_version=1)
def testGetRulesMatchingQueryVersionGlobbing(self):
expected = [dict(rule_id=1, priority=100, throttle=100, version='4.0*', update_type='z', data_version=1)]
rules = self.rules.getRulesMatchingQuery(
dict(name='', product='', version='4.0', channel='',
buildTarget='', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
self.assertEquals(rules, expected)
rules = self.rules.getRulesMatchingQuery(
dict(name='', product='', version='4.0b2', channel='',
buildTarget='', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
self.assertEquals(rules, expected)
rules = self.rules.getRulesMatchingQuery(
dict(name='', product='', version='4.0.1', channel='',
buildTarget='', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
self.assertEquals(rules, expected)
def testGetRulesMatchingQueryChannelGlobbing(self):
expected = [dict(rule_id=2, priority=100, throttle=100, channel='release*', update_type='z', data_version=1)]
rules = self.rules.getRulesMatchingQuery(
dict(name='', product='', version='', channel='releasetest',
buildTarget='', buildID='', locale='', osVersion='', distribution='',
distVersion='', headerArchitecture=''
),
fallbackChannel='releasetest'
)
rules = self._stripNullColumns(rules)
self.assertEquals(rules, expected)
rules = self.rules.getRulesMatchingQuery(
dict(name='', product='', version='', channel='releasetest-cck-blah',
buildTarget='', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel='releasetest'
)
rules = self._stripNullColumns(rules)
self.assertEquals(rules, expected)
class TestReleases(unittest.TestCase, MemoryDatabaseMixin):
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.create()
self.releases = self.db.releases
self.releases.t.insert().execute(name='a', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1)
self.releases.t.insert().execute(name='ab', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1)
self.releases.t.insert().execute(name='b', product='b', version='b', data=json.dumps(dict(name=2)), data_version=1)
self.releases.t.insert().execute(name='c', product='c', version='c', data=json.dumps(dict(name=3)), data_version=1)
def testGetReleases(self):
self.assertEquals(len(self.releases.getReleases()), 4)
def testGetReleasesWithLimit(self):
self.assertEquals(len(self.releases.getReleases(limit=1)), 1)
def testGetReleasesWithWhere(self):
expected = [dict(product='b', version='b', name='b', data=dict(name=2), data_version=1)]
self.assertEquals(self.releases.getReleases(name='b'), expected)
def testGetReleaseBlob(self):
expected = dict(name=3)
self.assertEquals(self.releases.getReleaseBlob(name='c'), expected)
def testGetReleaseBlobNonExistentRelease(self):
self.assertRaises(KeyError, self.releases.getReleaseBlob, name='z')
def testGetReleaseNames(self):
releases = self.releases.getReleaseNames()
expected = [ dict(name='a'),
dict(name='ab'),
dict(name='b'),
dict(name='c')]
self.assertEquals(releases, expected)
releases = self.releases.getReleaseNames(product='a')
expected = [ dict(name='a'),
dict(name='ab')]
self.assertEquals(releases, expected)
releases = self.releases.getReleaseNames(version='b')
expected = [ dict(name='b'), ]
self.assertEquals(releases, expected)
releases = self.releases.getReleaseNames(product='a', version='b')
expected = [ ]
self.assertEquals(releases, expected)
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.rules = self.db.rules
self.rules.t.insert().execute(id=1, priority=100, version='4.0*', throttle=100, update_type='z', data_version=1)
self.rules.t.insert().execute(id=2, priority=100, channel='release*', throttle=100, update_type='z', data_version=1)
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.releases = self.db.releases
self.releases.t.insert().execute(name='a', product='a', version='a', data_version=1, data="""
{
"name": "a",
"platforms": {
"p": {
"locales": {
"l": {
"complete": {
"filesize": "1234"
}
}
}
}
}
}
""")
self.releases.t.insert().execute(name='b', product='b', version='b', data_version=1, data="""
{
"name": "b"
}
""")
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.releases = self.db.releases
self.releases.t.insert().execute(name='a', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1)
self.releases.t.insert().execute(name='ab', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1)
self.releases.t.insert().execute(name='b', product='b', version='b', data=json.dumps(dict(name=2)), data_version=1)
self.releases.t.insert().execute(name='c', product='c', version='c', data=json.dumps(dict(name=3)), data_version=1)
def testReset(self):
db = AUSDatabase('sqlite:///:memory:')
db.reset()
# If we can set the dburi again, reset worked!
db.setDburi('sqlite:///:memory:')
db.createTables()
insp = Inspector.from_engine(db.engine)
self.assertNotEqual(insp.get_table_names(), [])
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.paths = self.db.rules
self.paths.t.insert().execute(id=1, priority=100, version='3.5', buildTarget='d', throttle=100, mapping='c', update_type='z', data_version=1)
self.paths.t.insert().execute(id=2, priority=100, version='3.3', buildTarget='d', throttle=100, mapping='b', update_type='z', data_version=1)
self.paths.t.insert().execute(id=3, priority=100, version='3.5', buildTarget='a', throttle=100, mapping='a', update_type='z', data_version=1)
self.paths.t.insert().execute(id=4, priority=80, buildTarget='d', throttle=100, mapping='a', update_type='z', data_version=1)
self.paths.t.insert().execute(id=5, priority=80, buildTarget='d', version='3.3', throttle=0, mapping='c', update_type='z', data_version=1)
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.permissions = self.db.permissions
self.permissions.t.insert().execute(permission='admin', username='******', data_version=1)
self.permissions.t.insert().execute(permission='/users/:id/permissions/:permission', username='******', data_version=1)
self.permissions.t.insert().execute(permission='/releases/:name', username='******', options=json.dumps(dict(product='fake')), data_version=1)
self.permissions.t.insert().execute(permission='/rules', username='******', data_version=1)
self.permissions.t.insert().execute(permission='/rules/:id', username='******', options=json.dumps(dict(method='POST')), data_version=1)
class TestReleases(unittest.TestCase, MemoryDatabaseMixin):
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.releases = self.db.releases
self.releases.t.insert().execute(name='a', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1)
self.releases.t.insert().execute(name='ab', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1)
self.releases.t.insert().execute(name='b', product='b', version='b', data=json.dumps(dict(name=2)), data_version=1)
self.releases.t.insert().execute(name='c', product='c', version='c', data=json.dumps(dict(name=3)), data_version=1)
def testGetReleases(self):
self.assertEquals(len(self.releases.getReleases()), 4)
def testGetReleasesWithLimit(self):
self.assertEquals(len(self.releases.getReleases(limit=1)), 1)
def testGetReleasesWithWhere(self):
expected = [dict(product='b', version='b', name='b', data=dict(name=2), data_version=1)]
self.assertEquals(self.releases.getReleases(name='b'), expected)
def testGetReleaseBlob(self):
expected = dict(name=3)
self.assertEquals(self.releases.getReleaseBlob(name='c'), expected)
def testGetReleaseBlobNonExistentRelease(self):
self.assertRaises(KeyError, self.releases.getReleaseBlob, name='z')
def testAddRelease(self):
blob = ReleaseBlobV1(name=4)
self.releases.addRelease(name='d', product='d', version='d', blob=blob, changed_by='bill')
expected = [('d', 'd', 'd', json.dumps(dict(name=4)), 1)]
self.assertEquals(self.releases.t.select().where(self.releases.name=='d').execute().fetchall(), expected)
def testAddReleaseAlreadyExists(self):
blob = ReleaseBlobV1(name=1)
self.assertRaises(TransactionError, self.releases.addRelease, name='a', product='a', version='a', blob=blob, changed_by='bill')
def testUpdateRelease(self):
self.releases.updateRelease(name='b', product='z', version='y', changed_by='bill', old_data_version=1)
expected = [('b', 'z', 'y', json.dumps(dict(name=2)), 2)]
self.assertEquals(self.releases.t.select().where(self.releases.name=='b').execute().fetchall(), expected)
def setDb(self,
dburi,
releases_history_buckets=None,
releases_history_class=None):
from auslib.db import AUSDatabase, GCSHistory
if not releases_history_class and releases_history_buckets is not None:
releases_history_class = GCSHistory
self.db = AUSDatabase(
dburi,
releases_history_buckets=releases_history_buckets,
releases_history_class=releases_history_class)
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument("--db", dest="db", help="The database to use, in URI format. Incompatible with --ini.")
parser.add_argument("--ini", dest="ini", help="The config file to look for the database in. Incompatible with --db.")
parser.add_argument("--name", dest="name", required=True, help="The name of the person/script doing the migrations.")
parser.add_argument("releases", metavar="release", nargs="+", help="The releases (aka blob names) to migrate.")
args = parser.parse_args()
if args.db and args.ini:
parser.error("Cannot specify --db and --ini!")
if args.ini:
cfg = AdminConfig(args.ini)
db = AUSDatabase(cfg.getDburi())
db.setDomainWhitelist(cfg.getDomainWhitelist())
else:
db = AUSDatabase(args.db)
for release in args.releases:
try:
rel = db.releases.getReleases(name=release)[0]
blob = rel["data"]
except:
log.debug("No such release '%s', skipping", release)
if blob["schema_version"] == 4:
log.debug("%s is already schema 4, skipping", release)
continue
elif blob["schema_version"] != 3:
def setDb(self, dburi):
from auslib.db import AUSDatabase
self.db = AUSDatabase(dburi)
"Anything older than this will be deleted.\n"
)
usage += " cleanup-dryrun: Show what would be removed if 'cleanup' is run."
parser = OptionParser(usage=usage)
parser.add_option("-d", "--db", dest="db", default=None, help="database to manage, in URI format")
parser.add_option("--version", dest="version", default=None, type="int", help="Create/upgrade to this specific schema version rather than the latest.")
options, args = parser.parse_args()
if not options.db:
parser.error("db is required")
if len(args) < 1:
parser.error("need an action to perform")
action = args[0]
db = AUSDatabase(options.db, mysql_traditional_mode=True)
if action == "create":
db.create(options.version)
elif action == "upgrade":
db.upgrade(options.version)
elif action == "downgrade":
db.downgrade(options.version)
elif action == "extract":
with db.begin() as trans:
if len(args) < 2:
extract_active_data(trans, options.db)
else:
location = args[1]
extract_active_data(trans, options.db, location)
elif action.startswith("cleanup"):
if len(args) < 2:
if __name__ == "__main__":
from optparse import OptionParser
usage = """%s --db dburi action\n""" % sys.argv[0]
usage += "Possible actions:\n"
usage += " create: Create all the tables required for a new Balrog database\n"
usage += " upgrade: Upgrade an existing balrog table to a newer version."
parser = OptionParser(usage=usage)
parser.add_option("-d", "--db", dest="db", default=None, help="database to manage, in URI format")
parser.add_option(
"--version",
dest="version",
default=None,
type="int",
help="Create/upgrade to this specific schema version rather than the latest.",
)
options, args = parser.parse_args()
if not options.db:
parser.error("db is required")
if len(args) != 1:
parser.error("need a single action to perform")
action = args[0]
db = AUSDatabase(options.db)
if action == "create":
db.create(options.version)
elif action == "upgrade":
db.upgrade(options.version)
dest="name",
required=True,
help="The name of the person/script doing the migrations.")
parser.add_argument("releases",
metavar="release",
nargs="+",
help="The releases (aka blob names) to migrate.")
args = parser.parse_args()
if args.db and args.ini:
parser.error("Cannot specify --db and --ini!")
if args.ini:
cfg = AdminConfig(args.ini)
db = AUSDatabase(cfg.getDburi())
db.setDomainWhitelist(cfg.getDomainWhitelist())
else:
db = AUSDatabase(args.db)
for release in args.releases:
try:
rel = db.releases.getReleases(name=release)[0]
blob = rel["data"]
except:
log.debug("No such release '%s', skipping", release)
if blob["schema_version"] == 4:
log.debug("%s is already schema 4, skipping", release)
continue
elif blob["schema_version"] != 3:
class TestReleasesSchema1(unittest.TestCase, MemoryDatabaseMixin):
"""Tests for the Releases class that depend on version 1 of the blob schema."""
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.create()
self.releases = self.db.releases
self.releases.t.insert().execute(name='a', product='a', version='a', data_version=1, data="""
{
"name": "a",
"platforms": {
"p": {
"locales": {
"l": {
"complete": {
"filesize": 1234
}
}
}
},
"p2": {
"alias": "p"
}
}
}
""")
self.releases.t.insert().execute(name='b', product='b', version='b', data_version=1, data="""
{
"name": "b"
}
""")
def testAddRelease(self):
blob = ReleaseBlobV1(name=4)
self.releases.addRelease(name='d', product='d', version='d', blob=blob, changed_by='bill')
expected = [('d', 'd', 'd', json.dumps(dict(name=4)), 1)]
self.assertEquals(self.releases.t.select().where(self.releases.name=='d').execute().fetchall(), expected)
def testAddReleaseAlreadyExists(self):
blob = ReleaseBlobV1(name=1)
self.assertRaises(TransactionError, self.releases.addRelease, name='a', product='a', version='a', blob=blob, changed_by='bill')
def testUpdateRelease(self):
blob = ReleaseBlobV1(name='a')
self.releases.updateRelease(name='b', product='z', version='y', blob=blob, changed_by='bill', old_data_version=1)
expected = [('b', 'z', 'y', json.dumps(dict(name='a')), 2)]
self.assertEquals(self.releases.t.select().where(self.releases.name=='b').execute().fetchall(), expected)
def testUpdateReleaseWithBlob(self):
blob = ReleaseBlobV1(name='b', schema_version=3)
self.releases.updateRelease(name='b', product='z', version='y', changed_by='bill', blob=blob, old_data_version=1)
expected = [('b', 'z', 'y', json.dumps(dict(name='b', schema_version=3)), 2)]
self.assertEquals(self.releases.t.select().where(self.releases.name=='b').execute().fetchall(), expected)
def testUpdateReleaseInvalidBlob(self):
blob = ReleaseBlobV1(name=2)
blob['foo'] = 'bar'
self.assertRaises(ValueError, self.releases.updateRelease, changed_by='bill', name='b', blob=blob, old_data_version=1)
def testAddLocaleToRelease(self):
data = dict(complete=dict(hashValue='abc'))
self.releases.addLocaleToRelease(name='a', platform='p', locale='c', data=data, old_data_version=1, changed_by='bill')
ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0])
expected = json.loads("""
{
"name": "a",
"platforms": {
"p": {
"locales": {
"c": {
"complete": {
"hashValue": "abc"
}
},
"l": {
"complete": {
"filesize": 1234
}
}
}
},
"p2": {
"alias": "p"
}
}
}
""")
self.assertEqual(ret, expected)
def testAddLocaleToReleaseOverride(self):
data = dict(complete=dict(hashValue=789))
self.releases.addLocaleToRelease(name='a', platform='p', locale='l', data=data, old_data_version=1, changed_by='bill')
ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0])
expected = json.loads("""
{
"name": "a",
"platforms": {
"p": {
"locales": {
"l": {
"complete": {
"hashValue": 789
}
}
}
},
"p2": {
"alias": "p"
}
}
}
""")
self.assertEqual(ret, expected)
def testAddLocaleToReleasePlatformsDoesntExist(self):
data = dict(complete=dict(filesize=432))
self.releases.addLocaleToRelease(name='b', platform='q', locale='l', data=data, old_data_version=1, changed_by='bill')
ret = json.loads(select([self.releases.data]).where(self.releases.name=='b').execute().fetchone()[0])
expected = json.loads("""
{
"name": "b",
"platforms": {
"q": {
"locales": {
"l": {
"complete": {
"filesize": 432
}
}
}
}
}
}
""")
self.assertEqual(ret, expected)
def testAddLocaleToReleaseSecondPlatform(self):
data = dict(complete=dict(filesize=324))
self.releases.addLocaleToRelease(name='a', platform='q', locale='l', data=data, old_data_version=1, changed_by='bill')
ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0])
expected = json.loads("""
{
"name": "a",
"platforms": {
"p": {
"locales": {
"l": {
"complete": {
"filesize": 1234
}
}
}
},
"p2": {
"alias": "p"
},
"q": {
"locales": {
"l": {
"complete": {
"filesize": 324
}
}
}
}
}
}
""")
self.assertEqual(ret, expected)
def testAddLocaleToReleaseResolveAlias(self):
data = dict(complete=dict(filesize=444))
self.releases.addLocaleToRelease(name='a', platform='p2', locale='j', data=data, old_data_version=1, changed_by='bill')
ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0])
expected = json.loads("""
{
"name": "a",
"platforms": {
"p": {
"locales": {
"l": {
"complete": {
"filesize": 1234
}
},
"j": {
"complete": {
"filesize": 444
}
}
}
},
"p2": {
"alias": "p"
}
}
}
""")
self.assertEqual(ret, expected)
def testCreateTables(self):
db = AUSDatabase()
db.setDburi('sqlite:///:memory:')
db.createTables()
insp = Inspector.from_engine(db.engine)
self.assertNotEqual(insp.get_table_names(), [])
class TestRulesSimple(unittest.TestCase, RulesTestMixin, MemoryDatabaseMixin):
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.create()
self.paths = self.db.rules
self.paths.t.insert().execute(id=1, priority=100, version='3.5', buildTarget='d', throttle=100, mapping='c', update_type='z', data_version=1)
self.paths.t.insert().execute(id=2, priority=100, version='3.3', buildTarget='d', throttle=100, mapping='b', update_type='z', data_version=1)
self.paths.t.insert().execute(id=3, priority=100, version='3.5', buildTarget='a', throttle=100, mapping='a', update_type='z', data_version=1)
self.paths.t.insert().execute(id=4, priority=80, buildTarget='d', throttle=100, mapping='a', update_type='z', data_version=1)
self.paths.t.insert().execute(id=5, priority=80, buildTarget='d', version='3.3', throttle=0, mapping='c', update_type='z', data_version=1)
def testGetOrderedRules(self):
rules = self._stripNullColumns(self.paths.getOrderedRules())
expected = [
dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1),
dict(rule_id=5, priority=80, throttle=0, version='3.3', buildTarget='d', mapping='c', update_type='z', data_version=1),
dict(rule_id=2, priority=100, throttle=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1),
dict(rule_id=3, priority=100, throttle=100, version='3.5', buildTarget='a', mapping='a', update_type='z', data_version=1),
dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1),
]
self.assertEquals(rules, expected)
def testGetRulesMatchingQuery(self):
rules = self.paths.getRulesMatchingQuery(
dict(name='', product='', version='3.5', channel='',
buildTarget='a', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture='',
force=False
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
expected = [dict(rule_id=3, priority=100, throttle=100, version='3.5', buildTarget='a', mapping='a', update_type='z', data_version=1)]
self.assertEquals(rules, expected)
def testGetRulesMatchingQueryWithNullColumn(self):
rules = self.paths.getRulesMatchingQuery(
dict(name='', product='', version='3.5', channel='',
buildTarget='d', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture='',
force=False
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
expected = [
dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1),
dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1),
]
self.assertEquals(rules, expected)
def testGetRulesMatchingQueryDontReturnThrottled(self):
rules = self.paths.getRulesMatchingQuery(
dict(name='', product='', version='3.3', channel='',
buildTarget='d', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture='',
force=False
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
expected = [
dict(rule_id=2, priority=100, throttle=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1),
dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1),
]
self.assertEquals(rules, expected)
def testGetRulesMatchingQueryReturnThrottled(self):
rules = self.paths.getRulesMatchingQuery(
dict(name='', product='', version='3.3', channel='',
buildTarget='d', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture='',
force=True
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
expected = [
dict(rule_id=2, priority=100, throttle=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1),
dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1),
dict(rule_id=5, priority=80, throttle=0, version='3.3', buildTarget='d', mapping='c', update_type='z', data_version=1)
]
self.assertEquals(rules, expected)
def testGetRuleById(self):
rule = self._stripNullColumns([self.paths.getRuleById(1)])
expected = [dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1)]
self.assertEquals(rule, expected)
def testAddRule(self):
what = dict(throttle=11,
mapping='c',
update_type='z',
priority=60)
rule_id = self.paths.addRule(changed_by='bill', what=what)
rule_id = rule_id[0]
rules = self.paths.t.select().where(self.paths.rule_id==rule_id).execute().fetchall()
copy_rule = dict(rules[0].items())
rule = self._stripNullColumns( [copy_rule] )
what['rule_id']=rule_id
what['data_version']=1
what = [what]
self.assertEquals(rule, what)
def testUpdateRule(self):
rules = self.paths.t.select().where(self.paths.rule_id==1).execute().fetchall()
what = dict(rules[0].items())
what['mapping'] = 'd'
self.paths.updateRule(changed_by='bill', rule_id=1, what=what, old_data_version=1)
rules = self.paths.t.select().where(self.paths.rule_id==1).execute().fetchall()
copy_rule = dict(rules[0].items())
rule = self._stripNullColumns( [copy_rule] )
expected = [dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='d', update_type='z', data_version=1)]
self.assertEquals(rule, expected)
class AUS3:
def __init__(self, dbname=None):
self.setDb(dbname)
def setDb(self, dbname):
if dbname == None:
dbname = "sqlite:///update.db"
self.db = AUSDatabase(dbname)
self.releases = self.db.releases
self.rules = self.db.rules
def createTables(self):
self.db.createTables()
def identifyRequest(self, updateQuery):
log.debug("AUS.identifyRequest: got updateQuery: %s", updateQuery)
buildTarget = updateQuery["buildTarget"]
buildID = updateQuery["buildID"]
locale = updateQuery["locale"]
for release in self.releases.getReleases(product=updateQuery["product"], version=updateQuery["version"]):
log.debug("AUS.identifyRequest: Trying to match request to %s", release["name"])
if buildTarget in release["data"]["platforms"]:
releaseBuildID = release["data"].getBuildID(buildTarget, locale)
log.debug("AUS.identifyRequest: releasePlat buildID is: %s", releaseBuildID)
if buildID == releaseBuildID:
log.debug("AUS.identifyRequest: Identified query as %s", release["name"])
return release["name"]
log.debug("AUS.identifyRequest: Couldn't identify query")
return None
def evaluateRules(self, updateQuery):
log.debug("AUS.evaluateRules: Looking for rules that apply to:")
log.debug("AUS.evaluateRules: %s", updateQuery)
rules = self.rules.getRulesMatchingQuery(
updateQuery, fallbackChannel=self.getFallbackChannel(updateQuery["channel"])
)
### XXX throw any N->N update rules and keep the highest priority remaining one
if len(rules) >= 1:
rules = sorted(rules, key=lambda rule: rule["priority"], reverse=True)
log.debug("AUS.evaluateRules: Returning rule:")
log.debug("AUS.evaluateRules: %s", rules[0])
return rules[0]
return None
def getFallbackChannel(self, channel):
return channel.split("-cck-")[0]
def expandRelease(self, updateQuery, rule):
if not rule or not rule["mapping"]:
log.debug("AUS.expandRelease: Couldn't find rule or mapping for %s" % rule)
return None
# read data from releases table
try:
res = self.releases.getReleases(name=rule["mapping"], limit=1)[0]
except IndexError:
# need to log some sort of data inconsistency error here
log.debug("AUS.expandRelease: Failed to get release data from db for:")
log.debug("AUS.expandRelease: %s", rule["mapping"])
return None
relData = res["data"]
updateData = defaultdict(list)
# platforms may be aliased to another platform in the case
# of identical data, minimizing the json size
buildTarget = updateQuery["buildTarget"]
relDataPlat = relData.getPlatformData(buildTarget)
locale = updateQuery["locale"]
# return early if we don't have an update for this platform
if buildTarget not in relData["platforms"]:
log.debug("AUS.expandRelease: No platform %s in release %s", buildTarget, rule["mapping"])
return updateData
# return early if we don't have an update for this locale
if locale not in relDataPlat["locales"]:
log.debug("AUS.expandRelease: No update to %s for %s/%s", rule["mapping"], buildTarget, locale)
return updateData
else:
relDataPlatLoc = relDataPlat["locales"][locale]
# this is for the properties AUS2 can cope with today
if relData["schema_version"] == 1:
updateData["type"] = rule["update_type"]
updateData["appv"] = relData.getAppv(buildTarget, locale)
updateData["extv"] = relData.getExtv(buildTarget, locale)
updateData["schema_version"] = relData["schema_version"]
if "detailsUrl" in relData:
updateData["detailsUrl"] = relData["detailsUrl"].replace("%LOCALE%", updateQuery["locale"])
updateData["build"] = relData.getBuildID(buildTarget, locale)
# evaluate types of updates and see if we can use them
for patchKey in relDataPlatLoc:
if patchKey not in ("partial", "complete"):
log.debug("AUS.expandRelease: Skipping patchKey '%s'", patchKey)
continue
patch = relDataPlatLoc[patchKey]
if patch["from"] == updateQuery["name"] or patch["from"] == "*":
if "fileUrl" in patch:
url = patch["fileUrl"]
else:
# When we're using a fallback channel it's unlikely
# we'll have a fileUrl specifically for it, but we
# should try nonetheless. Non-fallback cases shouldn't
# be hitting any exceptions here.
try:
url = relData["fileUrls"][updateQuery["channel"]]
except KeyError:
url = relData["fileUrls"][self.getFallbackChannel(updateQuery["channel"])]
url = url.replace("%LOCALE%", updateQuery["locale"])
url = url.replace("%OS_FTP%", relDataPlat["OS_FTP"])
url = url.replace("%FILENAME%", relData["ftpFilenames"][patchKey])
url = url.replace("%PRODUCT%", relData["bouncerProducts"][patchKey])
url = url.replace("%OS_BOUNCER%", relDataPlat["OS_BOUNCER"])
updateData["patches"].append(
{
"type": patchKey,
"URL": url,
"hashFunction": relData["hashFunction"],
"hashValue": patch["hashValue"],
"size": patch["filesize"],
}
)
else:
log.debug(
"AUS.expandRelease: Didn't add patch for patchKey '%s'; from is '%s', updateQuery name is '%s'",
patchKey,
patch["from"],
updateQuery["name"],
)
# older branches required a <partial> in the update.xml, which we
# used to fake by repeating the complete data.
if (
"fakePartials" in relData
and relData["fakePartials"]
and len(updateData["patches"]) == 1
and updateData["patches"][0]["type"] == "complete"
):
patch = copy.copy(updateData["patches"][0])
patch["type"] = "partial"
updateData["patches"].append(patch)
log.debug("AUS.expandRelease: Returning %s", updateData)
return updateData
def createSnippet(self, updateQuery, release):
rel = self.expandRelease(updateQuery, release)
if not rel:
# handle this better, both for prod and debugging
log.debug("AUS.createSnippet: Couldn't expand rule for update target")
# XXX: Not sure we should be specifying patch types here, but it's
# required for tests that have null snippets in them at the time
# of writing.
return {"partial": "", "complete": ""}
snippets = {}
for patch in rel["patches"]:
snippet = [
"version=1",
"type=%s" % patch["type"],
"url=%s" % patch["URL"],
"hashFunction=%s" % patch["hashFunction"],
"hashValue=%s" % patch["hashValue"],
"size=%s" % patch["size"],
"build=%s" % rel["build"],
"appv=%s" % rel["appv"],
"extv=%s" % rel["extv"],
]
if rel["detailsUrl"]:
snippet.append("detailsUrl=%s" % rel["detailsUrl"])
if rel["type"] == "major":
snippets.append("updateType=major")
# AUS2 snippets have a trailing newline, add one here for easy diffing
snippets[patch["type"]] = "\n".join(snippet) + "\n"
# XXX: need to handle old releases needing completes duplicating partials
# add another parameter in the rule table and use it here
return snippets
def createXML(self, updateQuery, release):
rel = self.expandRelease(updateQuery, release)
# this will fall down all sorts of interesting ways by hardcoding fields
xml = ['<?xml version="1.0"?>']
xml.append("<updates>")
if rel:
if rel["schema_version"] == 1:
xml.append(
' <update type="%s" version="%s" extensionVersion="%s" buildID="%s"'
% (rel["type"], rel["appv"], rel["extv"], rel["build"])
)
if rel["detailsUrl"]:
xml.append(' detailsURL="%s"' % rel["detailsUrl"])
xml.append(">")
for patch in sorted(rel["patches"]):
xml.append(
' <patch type="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s"/>'
% (patch["type"], patch["URL"], patch["hashFunction"], patch["hashValue"], patch["size"])
)
# XXX: need to handle old releases needing completes duplicating partials
# add another parameter in the rule table and use it here
xml.append(" </update>")
xml.append("</updates>")
return "\n".join(xml)
sys.path.append(path.join(path.dirname(__file__), ".."))
from auslib.blobs.apprelease import ReleaseBlobV1
from auslib.db import AUSDatabase
if __name__ == "__main__":
from optparse import OptionParser
doc = "%s --db dburi -r release-name -v version -p product foo.json" % sys.argv[0]
parser = OptionParser(doc)
parser.add_option("-d", "--db", dest="db", default=None, help="database to manage, in URI format")
parser.add_option("-r", "--release", dest="release", default=None, help="Release to put blob into")
parser.add_option("-v", "--version", dest="version", default=None, help="Version of the release")
parser.add_option("-p", "--product", dest="product", default=None, help="Product of the release")
options, args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s: %(message)s")
if not options.db or not options.release or not options.version or not options.product or len(args) != 1:
print doc
sys.exit(1)
db = AUSDatabase(options.db)
blob = ReleaseBlobV1()
blob.loadJSON(open(args[0]).read())
try:
old = db.releases.getReleases(name=options.release)[0]
db.releases.updateRelease(name=options.release, product=options.product, version=options.version, changed_by='import-json', old_data_version=old['data_version'], blob=blob)
except IndexError:
db.releases.addRelease(name=options.release, product=options.product, version=options.version, blob=blob, changed_by='import-json')
class TestReleasesSchema1(unittest.TestCase, MemoryDatabaseMixin):
"""Tests for the Releases class that depend on version 1 of the blob schema."""
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.releases = self.db.releases
self.releases.t.insert().execute(name='a', product='a', version='a', data_version=1, data="""
{
"name": "a",
"platforms": {
"p": {
"locales": {
"l": {
"complete": {
"filesize": "1234"
}
}
}
}
}
}
""")
self.releases.t.insert().execute(name='b', product='b', version='b', data_version=1, data="""
{
"name": "b"
}
""")
def testAddLocaleToRelease(self):
blob = dict(complete=dict(hashValue='abc'))
self.releases.addLocaleToRelease(name='a', platform='p', locale='c', blob=blob, old_data_version=1, changed_by='bill')
ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0])
expected = json.loads("""
{
"name": "a",
"platforms": {
"p": {
"locales": {
"c": {
"complete": {
"hashValue": "abc"
}
},
"l": {
"complete": {
"filesize": "1234"
}
}
}
}
}
}
""")
self.assertEqual(ret, expected)
def testAddLocaleToReleaseOverride(self):
blob = dict(complete=dict(hashValue=789))
self.releases.addLocaleToRelease(name='a', platform='p', locale='l', blob=blob, old_data_version=1, changed_by='bill')
ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0])
expected = json.loads("""
{
"name": "a",
"platforms": {
"p": {
"locales": {
"l": {
"complete": {
"hashValue": 789
}
}
}
}
}
}
""")
self.assertEqual(ret, expected)
def testAddLocaleToReleasePlatformsDoesntExist(self):
blob = dict(complete=dict(filesize=432))
self.releases.addLocaleToRelease(name='b', platform='q', locale='l', blob=blob, old_data_version=1, changed_by='bill')
ret = json.loads(select([self.releases.data]).where(self.releases.name=='b').execute().fetchone()[0])
expected = json.loads("""
{
"name": "b",
"platforms": {
"q": {
"locales": {
"l": {
"complete": {
"filesize": 432
}
}
}
}
}
}
""")
self.assertEqual(ret, expected)
class TestRulesSimple(unittest.TestCase, RulesTestMixin, MemoryDatabaseMixin):
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.paths = self.db.rules
self.paths.t.insert().execute(id=1, priority=100, version='3.5', buildTarget='d', throttle=100, mapping='c', update_type='z', data_version=1)
self.paths.t.insert().execute(id=2, priority=100, version='3.3', buildTarget='d', throttle=100, mapping='b', update_type='z', data_version=1)
self.paths.t.insert().execute(id=3, priority=100, version='3.5', buildTarget='a', throttle=100, mapping='a', update_type='z', data_version=1)
self.paths.t.insert().execute(id=4, priority=80, buildTarget='d', throttle=100, mapping='a', update_type='z', data_version=1)
self.paths.t.insert().execute(id=5, priority=80, buildTarget='d', version='3.3', throttle=0, mapping='c', update_type='z', data_version=1)
def testGetOrderedRules(self):
rules = self._stripNullColumns(self.paths.getOrderedRules())
expected = [
dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1),
dict(rule_id=5, priority=80, throttle=0, version='3.3', buildTarget='d', mapping='c', update_type='z', data_version=1),
dict(rule_id=2, priority=100, throttle=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1),
dict(rule_id=3, priority=100, throttle=100, version='3.5', buildTarget='a', mapping='a', update_type='z', data_version=1),
dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1),
]
self.assertEquals(rules, expected)
def testGetRulesMatchingQuery(self):
rules = self.paths.getRulesMatchingQuery(
dict(name='', product='', version='3.5', channel='',
buildTarget='a', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
expected = [dict(rule_id=3, priority=100, throttle=100, version='3.5', buildTarget='a', mapping='a', update_type='z', data_version=1)]
self.assertEquals(rules, expected)
def testGetRulesMatchingQueryWithNullColumn(self):
rules = self.paths.getRulesMatchingQuery(
dict(name='', product='', version='3.5', channel='',
buildTarget='d', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
expected = [
dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1),
dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1),
]
self.assertEquals(rules, expected)
def testGetRulesMatchingQueryDontReturnThrottled(self):
rules = self.paths.getRulesMatchingQuery(
dict(name='', product='', version='3.3', channel='',
buildTarget='d', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
expected = [
dict(rule_id=2, priority=100, throttle=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1),
dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1),
]
self.assertEquals(rules, expected)
usage += " cleanup: Cleanup old data from a database. Requires an extra arg of maximum age (in days) of nightly releases. " \
"Anything older than this will be deleted.\n"
usage += " cleanup-dryrun: Show what would be removed if 'cleanup' is run."
parser = OptionParser(usage=usage)
parser.add_option("-d", "--db", dest="db", default=None, help="database to manage, in URI format")
parser.add_option("--version", dest="version", default=None, type="int", help="Create/upgrade to this specific schema version rather than the latest.")
options, args = parser.parse_args()
if not options.db:
parser.error("db is required")
if len(args) < 1:
parser.error("need an action to perform")
action = args[0]
db = AUSDatabase(options.db, mysql_traditional_mode=True)
if action == 'create':
db.create(options.version)
elif action == 'upgrade':
db.upgrade(options.version)
elif action == 'downgrade':
db.downgrade(options.version)
elif action.startswith("cleanup"):
if len(args) < 2:
parser.error("need to pass maximum nightly release age")
nightly_age = int(args[1])
with db.begin() as trans:
if action == "cleanup":
cleanup_releases(trans, nightly_age, dryrun=False)
cleanup_releases_history(trans, dryrun=False)
else:
class TestPermissions(unittest.TestCase, MemoryDatabaseMixin):
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.permissions = self.db.permissions
self.permissions.t.insert().execute(permission='admin', username='******', data_version=1)
self.permissions.t.insert().execute(permission='/users/:id/permissions/:permission', username='******', data_version=1)
self.permissions.t.insert().execute(permission='/releases/:name', username='******', options=json.dumps(dict(product='fake')), data_version=1)
self.permissions.t.insert().execute(permission='/rules', username='******', data_version=1)
self.permissions.t.insert().execute(permission='/rules/:id', username='******', options=json.dumps(dict(method='POST')), data_version=1)
def testGrantPermissions(self):
query = self.permissions.t.select().where(self.permissions.username=='jess')
self.assertEquals(len(query.execute().fetchall()), 0)
self.permissions.grantPermission('bob', 'jess', '/rules/:id')
self.assertEquals(query.execute().fetchall(), [('/rules/:id', 'jess', None, 1)])
def testGrantPermissionsWithOptions(self):
self.permissions.grantPermission('bob', 'cathy', '/releases/:name', options=dict(product='SeaMonkey'))
query = self.permissions.t.select().where(self.permissions.username=='cathy')
query = query.where(self.permissions.permission=='/releases/:name')
self.assertEquals(query.execute().fetchall(), [('/releases/:name', 'cathy', json.dumps(dict(product='SeaMonkey')), 1)])
def testGrantPermissionsNotAllowed(self):
self.assertRaises(PermissionDeniedError, self.permissions.grantPermission,
'cathy', 'bob', '/rules/:id'
)
def testGrantPermissionsUnknownPermission(self):
self.assertRaises(ValueError, self.permissions.grantPermission,
'bob', 'bud', 'bad'
)
def testGrantPermissionsUnknownOption(self):
self.assertRaises(ValueError, self.permissions.grantPermission,
'bob', 'bud', '/rules/:id', dict(foo=1)
)
def testRevokePermission(self):
self.permissions.revokePermission(changed_by='bill', username='******', permission='/releases/:name',
old_data_version=1)
query = self.permissions.t.select().where(self.permissions.username=='bob')
query = query.where(self.permissions.permission=='/releases/:name')
self.assertEquals(len(query.execute().fetchall()), 0)
def testCanEditUsers(self):
self.assertTrue(self.permissions.canEditUsers('bill'))
self.assertTrue(self.permissions.canEditUsers('bob'))
def testCanEditUsersFalse(self):
self.assertFalse(self.permissions.canEditUsers('cathy'))
def testGetAllUsers(self):
self.assertEquals(self.permissions.getAllUsers(), ['bill', 'bob', 'cathy'])
def testGetUserPermissions(self):
expected = {'/users/:id/permissions/:permission': dict(options=None, data_version=1),
'/releases/:name': dict(options=dict(product='fake'), data_version=1),
'/rules/:id': dict(options=dict(method='POST'), data_version=1)}
self.assertEquals(self.permissions.getUserPermissions('bob'), expected)
def testGetOptions(self):
expected = dict(product='fake')
self.assertEquals(self.permissions.getOptions('bob', '/releases/:name'), expected)
def testGetOptionsPermissionDoesntExist(self):
self.assertRaises(ValueError, self.permissions.getOptions, 'fake', 'fake')
def testGetOptionsNoOptions(self):
self.assertEquals(self.permissions.getOptions('cathy', '/rules'), {})
def testHasUrlPermission(self):
self.assertTrue(self.permissions.hasUrlPermission('cathy', '/rules', 'PUT', dict(product='fake')))
def testHasUrlPermissionWithOption(self):
self.assertTrue(self.permissions.hasUrlPermission('bob', '/rules/:id', 'POST', dict(product='fake')))
def testHasUrlPermissionNotAllowed(self):
self.assertFalse(self.permissions.hasUrlPermission('cathy', '/rules/:id', 'DELETE', dict(product='fake')))
def testHasUrlPermissionNotAllowedWithOption(self):
self.assertFalse(self.permissions.hasUrlPermission('bob', '/rules/:id', 'DELETE', dict(product='fake')))
def testHasUrlPermissionWithProduct(self):
self.assertTrue(self.permissions.hasUrlPermission('bob', '/releases/:name', 'DELETE', dict(product='fake')))
def setDb(self, dbname):
if dbname == None:
dbname = "sqlite:///update.db"
self.db = AUSDatabase(dbname)
self.releases = self.db.releases
self.rules = self.db.rules
def testModelIsSameAsRepository(self):
db2 = AUSDatabase('sqlite:///' + self.getTempfile())
db2.create()
diff = migrate.versioning.api.compare_model_to_db(db2.engine, self.db.migrate_repo, self.db.metadata)
if diff:
self.fail(str(diff))