def testGetWithWorkingHours(self):
"""Tests get() outside of working hours."""
old = datetime.datetime(2014, 12, 10, 20, 00, 00, 000000)
now = datetime.datetime(2016, 9, 12, 0, 00, 00, 000000)
with mock.patch.object(datetime, 'datetime') as datetime_mock:
datetime_mock.utcnow.return_value = old
u1 = models.AppleSUSProduct(tracks=['unstable'],
product_id='fooid1',
manual_override=False)
u1.put()
u2 = models.AppleSUSProduct(
tracks=['unstable', applesus.common.TESTING],
product_id='fooid3',
manual_override=False)
u2.put()
with mock.patch.object(applesus.AppleSUSCatalogSync,
'_DeprecateOrphanedProducts'):
self.auto_promote.get(now=now)
u1 = models.AppleSUSProduct.get(u1.key())
u2 = models.AppleSUSProduct.get(u2.key())
self.assertTrue(applesus.common.TESTING not in u1.tracks)
self.assertTrue(applesus.common.STABLE not in u2.tracks)
def testGet(self, _):
"""Tests get() within working hours."""
old = datetime.datetime(2014, 12, 10, 20, 00, 00, 000000)
now = datetime.datetime(2016, 9, 10, 11, 00, 00, 000000)
with mock.patch.object(datetime, 'datetime') as datetime_mock:
datetime_mock.utcnow.return_value = old
promote_testing_product = models.AppleSUSProduct(
tracks=['unstable'], product_id='fooid', manual_override=False)
promote_testing_product.put()
promote_stable_product = models.AppleSUSProduct(
tracks=['unstable', applesus.common.TESTING],
product_id='fooid3',
manual_override=False)
promote_stable_product.put()
toonew_product = models.AppleSUSProduct(tracks=['unstable'],
product_id='fooid2',
manual_override=False)
toonew_product.put()
with mock.patch.object(
applesus.applesus,
'GenerateAppleSUSCatalogs') as generate_catalog_mock:
with mock.patch.object(
self.auto_promote,
'_NotifyAdminsOfAutoPromotions') as notify_mock:
self.auto_promote.get(now=now)
notify_mock.assert_called_once()
generate_catalog_mock.assert_has_calls([
mock.call(applesus.common.TESTING),
mock.call(applesus.common.STABLE)
],
any_order=True)
self.assertEqual(
2, len(applesus.models.AdminAppleSUSProductLog.all().fetch(None)))
promote_stable_product = models.AppleSUSProduct.get(
promote_stable_product.key())
promote_testing_product = models.AppleSUSProduct.get(
promote_testing_product.key())
toonew_product = models.AppleSUSProduct.get(toonew_product.key())
self.assertTrue(
applesus.common.TESTING in promote_testing_product.tracks)
self.assertTrue(
applesus.common.STABLE not in promote_testing_product.tracks)
self.assertTrue(applesus.common.TESTING not in toonew_product.tracks)
self.assertTrue(applesus.common.STABLE not in toonew_product.tracks)
self.assertTrue(
applesus.common.STABLE in promote_stable_product.tracks)
def testChangeProductForceInstall(self, *_):
product_id = 'pid'
models.AppleSUSProduct(key_name=product_id,
product_id=product_id,
manual_override=False).put()
force_install_after_date = (datetime.datetime.now() +
datetime.timedelta(days=4))
self.testapp.post('/admin/applesus/product/' + product_id, {
'force_install_after_date':
datetime.datetime.strftime(force_install_after_date,
'%Y-%m-%d %H:%M'),
},
status=httplib.OK)
product = models.AppleSUSProduct.get_by_key_name(product_id)
self.assertEquals(
datetime.datetime.strftime(force_install_after_date,
'%Y-%m-%dT%H:%M:00Z'),
product.force_install_after_date_str)
self.RunAllDeferredTasks()
mail_stub = self.testbed.get_stub('mail')
messages = mail_stub.get_sent_messages()
self.assertEqual(1, len(messages))
self.assertEqual(1,
len(models.AdminAppleSUSProductLog.all().fetch(None)))
def testDeprecateOrphanedProducts(self):
"""Tests _DeprecateOrphanedProducts() with deprecated & active products."""
self.stubs.Set(applesus.applesus, 'OS_VERSIONS',
frozenset(['10.9', '10.10', None]))
self.stubs.Set(applesus.common, 'TRACKS',
applesus.common.TRACKS + ['parseerror'])
for os_version in applesus.applesus.OS_VERSIONS:
for track in applesus.common.TRACKS + ['untouched']:
if not os_version:
continue
key = '%s_%s' % (os_version, track)
applesus.models.AppleSUSCatalog(key_name=key,
plist='fooplist-%s' %
key).put()
test_products = {
'10.8_unstable': ['product5', 'product6'],
'10.8_testing': ['product5'],
'10.8_stable': ['product5'],
'10.8_untouched': ['product7'],
'10.9_unstable': ['product5', 'product6'],
'10.9_testing': ['product5'],
'10.9_stable': ['product5'],
'10.9_untouched': ['product7'],
'10.10_unstable': ['product1'],
'10.10_testing': ['product3', 'product4'],
'10.10_stable': ['product3'],
'10.10_untouched': ['product7'],
'10.11_unstable': ['product1'],
'10.11_testing': ['product3', 'product4'],
'10.11_stable': ['product3'],
'10.11_untouched': ['product7'],
}
for p in ['product2', 'product3', 'product7', 'deprecateme', 'andme']:
models.AppleSUSProduct(product_id=p).put()
def PlistStub(plist):
m = mock.Mock()
if 'parseerror' in plist:
m.Parse.side_effect = applesus.plist.Error
else:
prefix = 'fooplist-'
assert plist.startswith(prefix)
m.get.return_value = test_products[plist[len(prefix):]]
return m
self.stubs.Set(applesus.plist, 'ApplePlist', PlistStub)
out = self.catalog_sync._DeprecateOrphanedProducts()
expected_deprecated = ['andme', 'deprecateme', 'product2']
self.assertEqual(expected_deprecated,
sorted(p.product_id for p in out))
products = models.AppleSUSProduct.all().filter('deprecated =', False)
self.assertEqual(['product3', 'product7'],
sorted(p.product_id for p in products))
def testDisplayUpdateLogs(self, render_mock, _):
product_id = 'pid'
p = models.AppleSUSProduct(key_name=product_id,
product_id=product_id,
tracks=['unstable'])
p.put()
models.AdminAppleSUSProductLog.Log(p, 'action description')
self.testapp.get('/admin/applesus/logs', {'product_id': product_id},
status=httplib.OK)
args = test.GetArgFromCallHistory(render_mock, arg_index=1)
self.assertEqual(1, len(args['logs']))
self.assertEqual(product_id, args['product_id'])
def testDisplayMain(self, render_mock, _):
product_id = 'pid'
models.AppleSUSProduct(key_name=product_id,
product_id=product_id,
tracks=['unstable']).put()
self.testapp.get('/admin/applesus/', status=httplib.OK)
args = test.GetArgFromCallHistory(render_mock, arg_index=1)
self.assertEqual(1, len(args['products']))
self.assertLess(args['products'][0].stable_promote_date,
(datetime.datetime.now() +
datetime.timedelta(days=60)).date())
def testChangeProductForceInstallToday(self, *_):
product_id = 'pid'
models.AppleSUSProduct(key_name=product_id,
product_id=product_id,
manual_override=False).put()
force_install_after_date = datetime.datetime.now()
self.testapp.post('/admin/applesus/product/' + product_id, {
'force_install_after_date':
datetime.datetime.strftime(force_install_after_date,
'%Y-%m-%d %H:%M'),
},
status=httplib.BAD_REQUEST)
def testChangeProductManualOverride(self, *_):
product_id = 'pid'
models.AppleSUSProduct(key_name=product_id,
product_id=product_id,
manual_override=False).put()
self.testapp.post('/admin/applesus/product/' + product_id, {
'manual_override': 1,
},
status=httplib.OK)
product = models.AppleSUSProduct.get_by_key_name(product_id)
self.assertTrue(product.manual_override)
self.RunAllDeferredTasks()
mail_stub = self.testbed.get_stub('mail')
messages = mail_stub.get_sent_messages()
self.assertEqual(1, len(messages))
self.assertEqual(1,
len(models.AdminAppleSUSProductLog.all().fetch(None)))
def testChangeProduct(self, *_):
settings.EMAIL_ON_EVERY_CHANGE = True
product_id = 'pid'
models.AppleSUSProduct(key_name=product_id,
product_id=product_id).put()
self.testapp.post('/admin/applesus/product/' + product_id, {
'enabled': 1,
'track': 'stable',
},
status=httplib.OK)
product = models.AppleSUSProduct.get_by_key_name(product_id)
self.assertEqual(['stable'], product.tracks)
self.RunAllDeferredTasks()
mail_stub = self.testbed.get_stub('mail')
messages = mail_stub.get_sent_messages()
self.assertEqual(1, len(messages))
self.assertEqual('*****@*****.**', messages[0].to)
def testUpdateProductDataFromCatalog(self, urlopen_mock,
dist_file_doc_mock):
"""Tests _UpdateProductDataFromCatalog()."""
product_one_id = '1productid'
product_one_url = 'http://example.com/%s.dist' % product_one_id
product_one_package_url = 'http://example.com/%s.pkg' % product_one_id
product_two_id = '2productid'
product_two_url = 'http://example.com/%s.dist' % product_one_id
product_two_package_url1 = 'http://example.com/%s-1.pkg' % product_two_id
product_two_package_url2 = 'http://example.com/%s-2.pkg' % product_two_id
product_two_dist = {
'version': 'twover',
'title': 'twotitle',
'description': 'twodesc',
'restart_required': False,
}
product_three_id = '3productid'
product_three_url = 'http://example.com/%s.dist' % product_three_id
product_three_package_url = 'http://example.com/%s.pkg' % product_three_id
product_three_dist = {
'version': 'threever',
'title': 'threetitle',
'description': 'threedesc',
'restart_required': True,
}
onedate = datetime.datetime(2014, 10, 8, 20, 00, 00, 000000)
twodate = datetime.datetime(2015, 10, 8, 12, 00, 00, 000000)
threedate = datetime.datetime(2013, 10, 8, 3, 00, 00, 000000)
catalog = {
'Products': {
product_one_id: {
'Distributions': {
'English': product_one_url
},
'PostDate': onedate,
'Packages': [{
'URL': [product_one_package_url]
}],
},
product_two_id: {
'Distributions': {
'English': product_two_url
},
'PostDate':
twodate,
'Packages': [{
'URL': product_two_package_url1
}, {
'URL': product_two_package_url2
}],
},
product_three_id: {
'Distributions': {
'en': product_three_url
},
'PostDate': threedate,
'Packages': [{
'URL': product_three_package_url
}],
},
}
}
mock_urllib_return = mock.Mock()
mock_urllib_return.code = httplib.OK # always return 200 for test.
mock_urllib_return.read.side_effect = [
product_two_dist, product_three_dist
]
urlopen_mock.return_value = mock_urllib_return
# product_one; add to existing_products so it's skipped.
models.AppleSUSProduct(product_id=product_one_id).put()
# product_two
mock_dfd_two = mock.Mock()
for k, v in product_two_dist.iteritems():
setattr(mock_dfd_two, k, v)
# product_three
mock_dfd_three = mock.Mock()
for k, v in product_three_dist.iteritems():
setattr(mock_dfd_three, k, v)
dist_file_doc_mock.side_effect = [mock_dfd_two, mock_dfd_three]
new_products = self.catalog_sync._UpdateProductDataFromCatalog(catalog)
urlopen_mock.assert_has_calls(
[mock.call(product_two_url),
mock.call(product_three_url)],
any_order=True)
product_two = models.AppleSUSProduct.all().filter(
'product_id =', product_two_id).fetch(1)[0]
product_three = models.AppleSUSProduct.all().filter(
'product_id =', product_three_id).fetch(1)[0]
self.assertEqual([product_two_id, product_three_id],
[p.product_id for p in new_products])
self.assertEqual(product_two.name, 'twotitle')
self.assertEqual(product_two.apple_mtime, twodate)
self.assertFalse(product_two.restart_required)
self.assertTrue(product_two.unattended)
self.assertEqual(product_two.package_urls,
[product_two_package_url1, product_two_package_url2])
self.assertEqual(product_three.version, 'threever')
self.assertEqual(product_three.description, 'threedesc')
self.assertTrue(product_three.restart_required)
self.assertFalse(product_three.unattended)
self.assertEqual(product_three.package_urls,
[product_three_package_url])
def _UpdateProductDataFromCatalog(cls, catalog_plist):
"""Updates models.AppleSUSProduct model from a catalog plist object.
Args:
catalog_plist: plist.ApplePlist object.
Returns:
list of new models.AppleSUSProduct objects, or empty list.
"""
if 'Products' not in catalog_plist:
logging.error('Products not found in Apple Updates catalog')
return []
new_products = []
# Create a dict of all previously processed product IDs for fast lookup.
existing_products = set()
products_query = models.AppleSUSProduct.all().filter(
'deprecated =', False)
for product in products_query:
existing_products.add(product.product_id)
# Loop over all products IDs in the Apple Updates catalog, adding any new
# products to the models.AppleSUSProduct model.
catalog_product_keys = catalog_plist.get('Products', {}).keys()
catalog_product_keys.sort()
for key in catalog_product_keys:
if key in existing_products:
continue # This product has already been processed in the past.
# Download and parse distribution metadata.
distributions = catalog_plist['Products'][key]['Distributions']
dist_url = distributions.get('English', None) or distributions.get(
'en', None)
if not dist_url:
logging.error(
'No english distributions exists for product %s; skipping.',
key)
continue # No english distribution exists :(
r = urllib2.urlopen(dist_url)
if r.code != httplib.OK:
continue
dist_str = r.read()
dist = applesus.DistFileDocument()
dist.LoadDocument(dist_str)
product = models.AppleSUSProduct(key_name=key)
product.product_id = key
product.name = dist.title
product.apple_mtime = catalog_plist['Products'][key]['PostDate']
product.version = dist.version
product.description = dist.description
product.tracks = [common.UNSTABLE]
product.restart_required = dist.restart_required
if not dist.restart_required and settings.APPLE_AUTO_UNATTENDED_ENABLED:
product.unattended = True
else:
product.unattended = False
# Parse package download URLs.
for package in catalog_plist['Products'][key]['Packages']:
product.package_urls.append(package.get('URL'))
product.put()
new_products.append(product)
return new_products
def _UpdateProductDataFromCatalog(self, catalog_plist):
"""Updates models.AppleSUSProduct model from a catalog plist object.
Args:
catalog_plist: plist.ApplePlist object.
Returns:
list of new models.AppleSUSProduct objects, or empty list.
"""
if 'Products' not in catalog_plist:
logging.error('Products not found in Apple Updates catalog')
return []
new_products = []
# Create a dict of all previously processed product IDs for fast lookup.
existing_products = {}
products_query = models.AppleSUSProduct.all()
for product in products_query:
existing_products[product.product_id] = True
# Loop over all products IDs in the Apple Updates catalog, adding any new
# products to the models.AppleSUSProduct model.
catalog_product_keys = catalog_plist.get('Products', {}).keys()
catalog_product_keys.sort()
for key in catalog_product_keys:
if key in existing_products:
continue # This product has already been processed in the past.
#logging.debug('Processing new product: %s', key)
distributions = catalog_plist['Products'][key]['Distributions']
url = distributions.get('English', None) or distributions.get(
'en', None)
if not url:
logging.error(
'No english distributions exists for product %s; skipping.',
key)
continue # No english distribution exists :(
r = urllib2.urlopen(url)
if r.code != 200:
#logging.warning('Skipping dist where HTTP status != 200')
continue
dist_str = r.read()
dist = applesus.DistFileDocument()
dist.LoadDocument(dist_str)
product = models.AppleSUSProduct(key_name=key)
product.product_id = key
product.name = dist.title
product.apple_mtime = catalog_plist['Products'][key]['PostDate']
product.version = dist.version
product.description = dist.description
product.tracks = [common.UNSTABLE]
product.restart_required = dist.restart_required
if not dist.restart_required and settings.APPLE_AUTO_UNATTENDED_ENABLED:
product.unattended = True
else:
product.unattended = False
product.put()
new_products.append(product)
#logging.debug('Product complete: %s', product.name)
return new_products
