class TestRulesSpecial(unittest.TestCase, RulesTestMixin, MemoryDatabaseMixin):
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.rules = self.db.rules
self.rules.t.insert().execute(id=1, priority=100, version='4.0*', throttle=100, update_type='z', data_version=1)
self.rules.t.insert().execute(id=2, priority=100, channel='release*', throttle=100, update_type='z', data_version=1)
def testGetRulesMatchingQueryVersionGlobbing(self):
expected = [dict(rule_id=1, priority=100, throttle=100, version='4.0*', update_type='z', data_version=1)]
rules = self.rules.getRulesMatchingQuery(
dict(name='', product='', version='4.0', channel='',
buildTarget='', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
self.assertEquals(rules, expected)
rules = self.rules.getRulesMatchingQuery(
dict(name='', product='', version='4.0b2', channel='',
buildTarget='', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
self.assertEquals(rules, expected)
rules = self.rules.getRulesMatchingQuery(
dict(name='', product='', version='4.0.1', channel='',
buildTarget='', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
self.assertEquals(rules, expected)
def testGetRulesMatchingQueryChannelGlobbing(self):
expected = [dict(rule_id=2, priority=100, throttle=100, channel='release*', update_type='z', data_version=1)]
rules = self.rules.getRulesMatchingQuery(
dict(name='', product='', version='', channel='releasetest',
buildTarget='', buildID='', locale='', osVersion='', distribution='',
distVersion='', headerArchitecture=''
),
fallbackChannel='releasetest'
)
rules = self._stripNullColumns(rules)
self.assertEquals(rules, expected)
rules = self.rules.getRulesMatchingQuery(
dict(name='', product='', version='', channel='releasetest-cck-blah',
buildTarget='', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel='releasetest'
)
rules = self._stripNullColumns(rules)
self.assertEquals(rules, expected)
def testReset(self):
db = AUSDatabase('sqlite:///:memory:')
db.reset()
# If we can set the dburi again, reset worked!
db.setDburi('sqlite:///:memory:')
db.createTables()
insp = Inspector.from_engine(db.engine)
self.assertNotEqual(insp.get_table_names(), [])
class TestReleases(unittest.TestCase, MemoryDatabaseMixin):
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.releases = self.db.releases
self.releases.t.insert().execute(name='a', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1)
self.releases.t.insert().execute(name='ab', product='a', version='a', data=json.dumps(dict(name=1)), data_version=1)
self.releases.t.insert().execute(name='b', product='b', version='b', data=json.dumps(dict(name=2)), data_version=1)
self.releases.t.insert().execute(name='c', product='c', version='c', data=json.dumps(dict(name=3)), data_version=1)
def testGetReleases(self):
self.assertEquals(len(self.releases.getReleases()), 4)
def testGetReleasesWithLimit(self):
self.assertEquals(len(self.releases.getReleases(limit=1)), 1)
def testGetReleasesWithWhere(self):
expected = [dict(product='b', version='b', name='b', data=dict(name=2), data_version=1)]
self.assertEquals(self.releases.getReleases(name='b'), expected)
def testGetReleaseBlob(self):
expected = dict(name=3)
self.assertEquals(self.releases.getReleaseBlob(name='c'), expected)
def testGetReleaseBlobNonExistentRelease(self):
self.assertRaises(KeyError, self.releases.getReleaseBlob, name='z')
def testAddRelease(self):
blob = ReleaseBlobV1(name=4)
self.releases.addRelease(name='d', product='d', version='d', blob=blob, changed_by='bill')
expected = [('d', 'd', 'd', json.dumps(dict(name=4)), 1)]
self.assertEquals(self.releases.t.select().where(self.releases.name=='d').execute().fetchall(), expected)
def testAddReleaseAlreadyExists(self):
blob = ReleaseBlobV1(name=1)
self.assertRaises(TransactionError, self.releases.addRelease, name='a', product='a', version='a', blob=blob, changed_by='bill')
def testUpdateRelease(self):
self.releases.updateRelease(name='b', product='z', version='y', changed_by='bill', old_data_version=1)
expected = [('b', 'z', 'y', json.dumps(dict(name=2)), 2)]
self.assertEquals(self.releases.t.select().where(self.releases.name=='b').execute().fetchall(), expected)
class AUS3:
def __init__(self, dbname=None):
self.setDb(dbname)
def setDb(self, dbname):
if dbname == None:
dbname = "sqlite:///update.db"
self.db = AUSDatabase(dbname)
self.releases = self.db.releases
self.rules = self.db.rules
def createTables(self):
self.db.createTables()
def identifyRequest(self, updateQuery):
log.debug("AUS.identifyRequest: got updateQuery: %s", updateQuery)
buildTarget = updateQuery["buildTarget"]
buildID = updateQuery["buildID"]
locale = updateQuery["locale"]
for release in self.releases.getReleases(product=updateQuery["product"], version=updateQuery["version"]):
log.debug("AUS.identifyRequest: Trying to match request to %s", release["name"])
if buildTarget in release["data"]["platforms"]:
releaseBuildID = release["data"].getBuildID(buildTarget, locale)
log.debug("AUS.identifyRequest: releasePlat buildID is: %s", releaseBuildID)
if buildID == releaseBuildID:
log.debug("AUS.identifyRequest: Identified query as %s", release["name"])
return release["name"]
log.debug("AUS.identifyRequest: Couldn't identify query")
return None
def evaluateRules(self, updateQuery):
log.debug("AUS.evaluateRules: Looking for rules that apply to:")
log.debug("AUS.evaluateRules: %s", updateQuery)
rules = self.rules.getRulesMatchingQuery(
updateQuery, fallbackChannel=self.getFallbackChannel(updateQuery["channel"])
)
### XXX throw any N->N update rules and keep the highest priority remaining one
if len(rules) >= 1:
rules = sorted(rules, key=lambda rule: rule["priority"], reverse=True)
log.debug("AUS.evaluateRules: Returning rule:")
log.debug("AUS.evaluateRules: %s", rules[0])
return rules[0]
return None
def getFallbackChannel(self, channel):
return channel.split("-cck-")[0]
def expandRelease(self, updateQuery, rule):
if not rule or not rule["mapping"]:
log.debug("AUS.expandRelease: Couldn't find rule or mapping for %s" % rule)
return None
# read data from releases table
try:
res = self.releases.getReleases(name=rule["mapping"], limit=1)[0]
except IndexError:
# need to log some sort of data inconsistency error here
log.debug("AUS.expandRelease: Failed to get release data from db for:")
log.debug("AUS.expandRelease: %s", rule["mapping"])
return None
relData = res["data"]
updateData = defaultdict(list)
# platforms may be aliased to another platform in the case
# of identical data, minimizing the json size
buildTarget = updateQuery["buildTarget"]
relDataPlat = relData.getPlatformData(buildTarget)
locale = updateQuery["locale"]
# return early if we don't have an update for this platform
if buildTarget not in relData["platforms"]:
log.debug("AUS.expandRelease: No platform %s in release %s", buildTarget, rule["mapping"])
return updateData
# return early if we don't have an update for this locale
if locale not in relDataPlat["locales"]:
log.debug("AUS.expandRelease: No update to %s for %s/%s", rule["mapping"], buildTarget, locale)
return updateData
else:
relDataPlatLoc = relDataPlat["locales"][locale]
# this is for the properties AUS2 can cope with today
if relData["schema_version"] == 1:
updateData["type"] = rule["update_type"]
updateData["appv"] = relData.getAppv(buildTarget, locale)
updateData["extv"] = relData.getExtv(buildTarget, locale)
updateData["schema_version"] = relData["schema_version"]
if "detailsUrl" in relData:
updateData["detailsUrl"] = relData["detailsUrl"].replace("%LOCALE%", updateQuery["locale"])
updateData["build"] = relData.getBuildID(buildTarget, locale)
# evaluate types of updates and see if we can use them
for patchKey in relDataPlatLoc:
if patchKey not in ("partial", "complete"):
log.debug("AUS.expandRelease: Skipping patchKey '%s'", patchKey)
continue
patch = relDataPlatLoc[patchKey]
if patch["from"] == updateQuery["name"] or patch["from"] == "*":
if "fileUrl" in patch:
url = patch["fileUrl"]
else:
# When we're using a fallback channel it's unlikely
# we'll have a fileUrl specifically for it, but we
# should try nonetheless. Non-fallback cases shouldn't
# be hitting any exceptions here.
try:
url = relData["fileUrls"][updateQuery["channel"]]
except KeyError:
url = relData["fileUrls"][self.getFallbackChannel(updateQuery["channel"])]
url = url.replace("%LOCALE%", updateQuery["locale"])
url = url.replace("%OS_FTP%", relDataPlat["OS_FTP"])
url = url.replace("%FILENAME%", relData["ftpFilenames"][patchKey])
url = url.replace("%PRODUCT%", relData["bouncerProducts"][patchKey])
url = url.replace("%OS_BOUNCER%", relDataPlat["OS_BOUNCER"])
updateData["patches"].append(
{
"type": patchKey,
"URL": url,
"hashFunction": relData["hashFunction"],
"hashValue": patch["hashValue"],
"size": patch["filesize"],
}
)
else:
log.debug(
"AUS.expandRelease: Didn't add patch for patchKey '%s'; from is '%s', updateQuery name is '%s'",
patchKey,
patch["from"],
updateQuery["name"],
)
# older branches required a <partial> in the update.xml, which we
# used to fake by repeating the complete data.
if (
"fakePartials" in relData
and relData["fakePartials"]
and len(updateData["patches"]) == 1
and updateData["patches"][0]["type"] == "complete"
):
patch = copy.copy(updateData["patches"][0])
patch["type"] = "partial"
updateData["patches"].append(patch)
log.debug("AUS.expandRelease: Returning %s", updateData)
return updateData
def createSnippet(self, updateQuery, release):
rel = self.expandRelease(updateQuery, release)
if not rel:
# handle this better, both for prod and debugging
log.debug("AUS.createSnippet: Couldn't expand rule for update target")
# XXX: Not sure we should be specifying patch types here, but it's
# required for tests that have null snippets in them at the time
# of writing.
return {"partial": "", "complete": ""}
snippets = {}
for patch in rel["patches"]:
snippet = [
"version=1",
"type=%s" % patch["type"],
"url=%s" % patch["URL"],
"hashFunction=%s" % patch["hashFunction"],
"hashValue=%s" % patch["hashValue"],
"size=%s" % patch["size"],
"build=%s" % rel["build"],
"appv=%s" % rel["appv"],
"extv=%s" % rel["extv"],
]
if rel["detailsUrl"]:
snippet.append("detailsUrl=%s" % rel["detailsUrl"])
if rel["type"] == "major":
snippets.append("updateType=major")
# AUS2 snippets have a trailing newline, add one here for easy diffing
snippets[patch["type"]] = "\n".join(snippet) + "\n"
# XXX: need to handle old releases needing completes duplicating partials
# add another parameter in the rule table and use it here
return snippets
def createXML(self, updateQuery, release):
rel = self.expandRelease(updateQuery, release)
# this will fall down all sorts of interesting ways by hardcoding fields
xml = ['<?xml version="1.0"?>']
xml.append("<updates>")
if rel:
if rel["schema_version"] == 1:
xml.append(
' <update type="%s" version="%s" extensionVersion="%s" buildID="%s"'
% (rel["type"], rel["appv"], rel["extv"], rel["build"])
)
if rel["detailsUrl"]:
xml.append(' detailsURL="%s"' % rel["detailsUrl"])
xml.append(">")
for patch in sorted(rel["patches"]):
xml.append(
' <patch type="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s"/>'
% (patch["type"], patch["URL"], patch["hashFunction"], patch["hashValue"], patch["size"])
)
# XXX: need to handle old releases needing completes duplicating partials
# add another parameter in the rule table and use it here
xml.append(" </update>")
xml.append("</updates>")
return "\n".join(xml)
def testCreateTables(self):
db = AUSDatabase()
db.setDburi('sqlite:///:memory:')
db.createTables()
insp = Inspector.from_engine(db.engine)
self.assertNotEqual(insp.get_table_names(), [])
class TestPermissions(unittest.TestCase, MemoryDatabaseMixin):
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.permissions = self.db.permissions
self.permissions.t.insert().execute(permission='admin', username='******', data_version=1)
self.permissions.t.insert().execute(permission='/users/:id/permissions/:permission', username='******', data_version=1)
self.permissions.t.insert().execute(permission='/releases/:name', username='******', options=json.dumps(dict(product='fake')), data_version=1)
self.permissions.t.insert().execute(permission='/rules', username='******', data_version=1)
self.permissions.t.insert().execute(permission='/rules/:id', username='******', options=json.dumps(dict(method='POST')), data_version=1)
def testGrantPermissions(self):
query = self.permissions.t.select().where(self.permissions.username=='jess')
self.assertEquals(len(query.execute().fetchall()), 0)
self.permissions.grantPermission('bob', 'jess', '/rules/:id')
self.assertEquals(query.execute().fetchall(), [('/rules/:id', 'jess', None, 1)])
def testGrantPermissionsWithOptions(self):
self.permissions.grantPermission('bob', 'cathy', '/releases/:name', options=dict(product='SeaMonkey'))
query = self.permissions.t.select().where(self.permissions.username=='cathy')
query = query.where(self.permissions.permission=='/releases/:name')
self.assertEquals(query.execute().fetchall(), [('/releases/:name', 'cathy', json.dumps(dict(product='SeaMonkey')), 1)])
def testGrantPermissionsNotAllowed(self):
self.assertRaises(PermissionDeniedError, self.permissions.grantPermission,
'cathy', 'bob', '/rules/:id'
)
def testGrantPermissionsUnknownPermission(self):
self.assertRaises(ValueError, self.permissions.grantPermission,
'bob', 'bud', 'bad'
)
def testGrantPermissionsUnknownOption(self):
self.assertRaises(ValueError, self.permissions.grantPermission,
'bob', 'bud', '/rules/:id', dict(foo=1)
)
def testRevokePermission(self):
self.permissions.revokePermission(changed_by='bill', username='******', permission='/releases/:name',
old_data_version=1)
query = self.permissions.t.select().where(self.permissions.username=='bob')
query = query.where(self.permissions.permission=='/releases/:name')
self.assertEquals(len(query.execute().fetchall()), 0)
def testCanEditUsers(self):
self.assertTrue(self.permissions.canEditUsers('bill'))
self.assertTrue(self.permissions.canEditUsers('bob'))
def testCanEditUsersFalse(self):
self.assertFalse(self.permissions.canEditUsers('cathy'))
def testGetAllUsers(self):
self.assertEquals(self.permissions.getAllUsers(), ['bill', 'bob', 'cathy'])
def testGetUserPermissions(self):
expected = {'/users/:id/permissions/:permission': dict(options=None, data_version=1),
'/releases/:name': dict(options=dict(product='fake'), data_version=1),
'/rules/:id': dict(options=dict(method='POST'), data_version=1)}
self.assertEquals(self.permissions.getUserPermissions('bob'), expected)
def testGetOptions(self):
expected = dict(product='fake')
self.assertEquals(self.permissions.getOptions('bob', '/releases/:name'), expected)
def testGetOptionsPermissionDoesntExist(self):
self.assertRaises(ValueError, self.permissions.getOptions, 'fake', 'fake')
def testGetOptionsNoOptions(self):
self.assertEquals(self.permissions.getOptions('cathy', '/rules'), {})
def testHasUrlPermission(self):
self.assertTrue(self.permissions.hasUrlPermission('cathy', '/rules', 'PUT', dict(product='fake')))
def testHasUrlPermissionWithOption(self):
self.assertTrue(self.permissions.hasUrlPermission('bob', '/rules/:id', 'POST', dict(product='fake')))
def testHasUrlPermissionNotAllowed(self):
self.assertFalse(self.permissions.hasUrlPermission('cathy', '/rules/:id', 'DELETE', dict(product='fake')))
def testHasUrlPermissionNotAllowedWithOption(self):
self.assertFalse(self.permissions.hasUrlPermission('bob', '/rules/:id', 'DELETE', dict(product='fake')))
def testHasUrlPermissionWithProduct(self):
self.assertTrue(self.permissions.hasUrlPermission('bob', '/releases/:name', 'DELETE', dict(product='fake')))
class TestReleasesSchema1(unittest.TestCase, MemoryDatabaseMixin):
"""Tests for the Releases class that depend on version 1 of the blob schema."""
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.releases = self.db.releases
self.releases.t.insert().execute(name='a', product='a', version='a', data_version=1, data="""
{
"name": "a",
"platforms": {
"p": {
"locales": {
"l": {
"complete": {
"filesize": "1234"
}
}
}
}
}
}
""")
self.releases.t.insert().execute(name='b', product='b', version='b', data_version=1, data="""
{
"name": "b"
}
""")
def testAddLocaleToRelease(self):
blob = dict(complete=dict(hashValue='abc'))
self.releases.addLocaleToRelease(name='a', platform='p', locale='c', blob=blob, old_data_version=1, changed_by='bill')
ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0])
expected = json.loads("""
{
"name": "a",
"platforms": {
"p": {
"locales": {
"c": {
"complete": {
"hashValue": "abc"
}
},
"l": {
"complete": {
"filesize": "1234"
}
}
}
}
}
}
""")
self.assertEqual(ret, expected)
def testAddLocaleToReleaseOverride(self):
blob = dict(complete=dict(hashValue=789))
self.releases.addLocaleToRelease(name='a', platform='p', locale='l', blob=blob, old_data_version=1, changed_by='bill')
ret = json.loads(select([self.releases.data]).where(self.releases.name=='a').execute().fetchone()[0])
expected = json.loads("""
{
"name": "a",
"platforms": {
"p": {
"locales": {
"l": {
"complete": {
"hashValue": 789
}
}
}
}
}
}
""")
self.assertEqual(ret, expected)
def testAddLocaleToReleasePlatformsDoesntExist(self):
blob = dict(complete=dict(filesize=432))
self.releases.addLocaleToRelease(name='b', platform='q', locale='l', blob=blob, old_data_version=1, changed_by='bill')
ret = json.loads(select([self.releases.data]).where(self.releases.name=='b').execute().fetchone()[0])
expected = json.loads("""
{
"name": "b",
"platforms": {
"q": {
"locales": {
"l": {
"complete": {
"filesize": 432
}
}
}
}
}
}
""")
self.assertEqual(ret, expected)
class TestRulesSimple(unittest.TestCase, RulesTestMixin, MemoryDatabaseMixin):
def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.createTables()
self.paths = self.db.rules
self.paths.t.insert().execute(id=1, priority=100, version='3.5', buildTarget='d', throttle=100, mapping='c', update_type='z', data_version=1)
self.paths.t.insert().execute(id=2, priority=100, version='3.3', buildTarget='d', throttle=100, mapping='b', update_type='z', data_version=1)
self.paths.t.insert().execute(id=3, priority=100, version='3.5', buildTarget='a', throttle=100, mapping='a', update_type='z', data_version=1)
self.paths.t.insert().execute(id=4, priority=80, buildTarget='d', throttle=100, mapping='a', update_type='z', data_version=1)
self.paths.t.insert().execute(id=5, priority=80, buildTarget='d', version='3.3', throttle=0, mapping='c', update_type='z', data_version=1)
def testGetOrderedRules(self):
rules = self._stripNullColumns(self.paths.getOrderedRules())
expected = [
dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1),
dict(rule_id=5, priority=80, throttle=0, version='3.3', buildTarget='d', mapping='c', update_type='z', data_version=1),
dict(rule_id=2, priority=100, throttle=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1),
dict(rule_id=3, priority=100, throttle=100, version='3.5', buildTarget='a', mapping='a', update_type='z', data_version=1),
dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1),
]
self.assertEquals(rules, expected)
def testGetRulesMatchingQuery(self):
rules = self.paths.getRulesMatchingQuery(
dict(name='', product='', version='3.5', channel='',
buildTarget='a', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
expected = [dict(rule_id=3, priority=100, throttle=100, version='3.5', buildTarget='a', mapping='a', update_type='z', data_version=1)]
self.assertEquals(rules, expected)
def testGetRulesMatchingQueryWithNullColumn(self):
rules = self.paths.getRulesMatchingQuery(
dict(name='', product='', version='3.5', channel='',
buildTarget='d', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
expected = [
dict(rule_id=1, priority=100, throttle=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1),
dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1),
]
self.assertEquals(rules, expected)
def testGetRulesMatchingQueryDontReturnThrottled(self):
rules = self.paths.getRulesMatchingQuery(
dict(name='', product='', version='3.3', channel='',
buildTarget='d', buildID='', locale='', osVersion='',
distribution='', distVersion='', headerArchitecture=''
),
fallbackChannel=''
)
rules = self._stripNullColumns(rules)
expected = [
dict(rule_id=2, priority=100, throttle=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1),
dict(rule_id=4, priority=80, throttle=100, buildTarget='d', mapping='a', update_type='z', data_version=1),
]
self.assertEquals(rules, expected)
