def setUp(self): super(TestRepackageJetpack, self).setUp() urllib_patch = mock.patch('files.tasks.urllib') self.urllib = urllib_patch.start() self.addCleanup(urllib_patch.stop) self.addon = Addon.objects.get(id=3615) # Match the guid from jetpack.xpi. self.addon.update(guid='jid0-ODIKJS9b4IT3H1NYlPKr0NDtLuE@jetpack') self.file = self.addon.current_version.all_files[0] eq_(self.file.status, amo.STATUS_PUBLIC) # Set up a temp file so urllib.urlretrieve works. self.xpi_path = os.path.join(settings.ROOT, 'apps/files/fixtures/files/jetpack.xpi') tmp_file = tempfile.NamedTemporaryFile(delete=False) tmp_file.write(open(self.xpi_path, 'rb').read()) tmp_file.flush() self.urllib.urlretrieve.return_value = (tmp_file.name, None) self.upgrader = JetpackUpgrader() settings.SEND_REAL_EMAIL = True self.uuid = uuid.uuid4().hex
def jetpack(request): upgrader = JetpackUpgrader() minver, maxver = upgrader.jetpack_versions() form = JetpackUpgradeForm(request.POST) if request.method == 'POST': if form.is_valid(): if 'minver' in request.POST: data = form.cleaned_data upgrader.jetpack_versions(data['minver'], data['maxver']) elif 'upgrade' in request.POST: if upgrader.version(maxver): start_upgrade(minver, maxver) elif 'cancel' in request.POST: upgrader.cancel() return redirect('zadmin.jetpack') else: messages.error(request, form.errors.as_text()) jetpacks = find_jetpacks(minver, maxver, from_builder_only=True) upgrading = upgrader.version() # Current Jetpack version upgrading to. repack_status = upgrader.files() # The files being repacked. show = request.GET.get('show', upgrading or minver) subset = filter( lambda f: not f.needs_upgrade and f.jetpack_version == show, jetpacks) need_upgrade = filter(lambda f: f.needs_upgrade, jetpacks) repacked = [] if upgrading: # Group the repacked files by status for this Jetpack upgrade. grouped_files = sorted_groupby(repack_status.values(), key=lambda f: f['status']) for group, rows in grouped_files: rows = sorted(list(rows), key=lambda f: f['file']) for idx, row in enumerate(rows): rows[idx]['file'] = File.objects.get(id=row['file']) repacked.append((group, rows)) groups = sorted_groupby(jetpacks, 'jetpack_version') by_version = dict((version, len(list(files))) for version, files in groups) return render( request, 'zadmin/jetpack.html', dict(form=form, upgrader=upgrader, by_version=by_version, upgrading=upgrading, need_upgrade=need_upgrade, subset=subset, show=show, repacked=repacked, repack_status=repack_status))
def jetpack(request): upgrader = JetpackUpgrader() minver, maxver = upgrader.jetpack_versions() form = JetpackUpgradeForm(request.POST) if request.method == "POST": if form.is_valid(): if "minver" in request.POST: data = form.cleaned_data upgrader.jetpack_versions(data["minver"], data["maxver"]) elif "upgrade" in request.POST: if upgrader.version(maxver): start_upgrade(minver, maxver) elif "cancel" in request.POST: upgrader.cancel() return redirect("zadmin.jetpack") else: messages.error(request, form.errors.as_text()) jetpacks = find_jetpacks(minver, maxver, from_builder_only=True) upgrading = upgrader.version() # Current Jetpack version upgrading to. repack_status = upgrader.files() # The files being repacked. show = request.GET.get("show", upgrading or minver) subset = filter(lambda f: not f.needs_upgrade and f.jetpack_version == show, jetpacks) need_upgrade = filter(lambda f: f.needs_upgrade, jetpacks) repacked = [] if upgrading: # Group the repacked files by status for this Jetpack upgrade. grouped_files = sorted_groupby(repack_status.values(), key=lambda f: f["status"]) for group, rows in grouped_files: rows = sorted(list(rows), key=lambda f: f["file"]) for idx, row in enumerate(rows): rows[idx]["file"] = File.objects.get(id=row["file"]) repacked.append((group, rows)) groups = sorted_groupby(jetpacks, "jetpack_version") by_version = dict((version, len(list(files))) for version, files in groups) return jingo.render( request, "zadmin/jetpack.html", dict( form=form, upgrader=upgrader, by_version=by_version, upgrading=upgrading, need_upgrade=need_upgrade, subset=subset, show=show, repacked=repacked, repack_status=repack_status, ), )
def setUp(self): super(TestUpgradeJetpacks, self).setUp() urllib2_patch = mock.patch('files.tasks.urllib2') self.urllib2 = urllib2_patch.start() self.addCleanup(urllib2_patch.stop) JetpackUpgrader().jetpack_versions('0.9', '1.0')
def check_jetpack_version(sender, **kw): import files.tasks from files.utils import JetpackUpgrader minver, maxver = JetpackUpgrader().jetpack_versions() qs = File.objects.filter(version__addon=sender, jetpack_version__isnull=False) ids = [f.id for f in qs if vint(minver) <= vint(f.jetpack_version) < vint(maxver)] if ids: files.tasks.start_upgrade.delay(ids, priority='high')
def setUp(self): super(TestRepackageJetpack, self).setUp() urllib_patch = mock.patch("files.tasks.urllib") self.urllib = urllib_patch.start() self.addCleanup(urllib_patch.stop) self.addon = Addon.objects.get(id=3615) # Match the guid from jetpack.xpi. self.addon.update(guid="jid0-ODIKJS9b4IT3H1NYlPKr0NDtLuE@jetpack") self.file = self.addon.current_version.all_files[0] eq_(self.file.status, amo.STATUS_PUBLIC) # Set up a temp file so urllib.urlretrieve works. self.xpi_path = os.path.join(settings.ROOT, "apps/files/fixtures/files/jetpack.xpi") self.tmp_file_path = self.create_temp_file() self.urllib.urlretrieve.return_value = (self.tmp_file_path, None) self.upgrader = JetpackUpgrader() settings.SEND_REAL_EMAIL = True self.uuid = uuid.uuid4().hex
def setUp(self): self.addon = Addon.objects.get(id=3615) JetpackUpgrader().jetpack_versions('1.0', '1.1')
class TestRepackageJetpack(amo.tests.TestCase): fixtures = ['base/addon_3615'] def setUp(self): super(TestRepackageJetpack, self).setUp() urllib_patch = mock.patch('files.tasks.urllib') self.urllib = urllib_patch.start() self.addCleanup(urllib_patch.stop) self.addon = Addon.objects.get(id=3615) # Match the guid from jetpack.xpi. self.addon.update(guid='jid0-ODIKJS9b4IT3H1NYlPKr0NDtLuE@jetpack') self.file = self.addon.current_version.all_files[0] eq_(self.file.status, amo.STATUS_PUBLIC) # Set up a temp file so urllib.urlretrieve works. self.xpi_path = os.path.join(settings.ROOT, 'apps/files/fixtures/files/jetpack.xpi') self.tmp_file_path = self.create_temp_file() self.urllib.urlretrieve.return_value = (self.tmp_file_path, None) self.upgrader = JetpackUpgrader() settings.SEND_REAL_EMAIL = True self.uuid = uuid.uuid4().hex def tearDown(self): storage.delete(self.tmp_file_path) def create_temp_file(self): path = tempfile.mktemp(dir=settings.TMP_PATH) with storage.open(path, 'w') as tmp_file: tmp_file.write(open(self.xpi_path, 'rb').read()) return path def builder_data(self, **kw): """Generate builder_data response dictionary with sensible defaults.""" # Tell Redis we're sending to builder. self.upgrader.file(self.file.id, {'uuid': self.uuid, 'file': self.file.id, 'owner': 'bulk', 'version': '1.0b4'}) request = {'uuid': self.uuid, 'addon': self.addon.id, 'file_id': self.file.id} request.update((k, v) for k, v in kw.items() if k in request) data = {'result': 'success', 'msg': 'ok', 'location': 'http://new.file', # This fakes what we would send to the builder. 'request': urllib.urlencode(request)} data.update(kw) return data def test_result_not_success(self): data = self.builder_data(result='fail', msg='oops') eq_(tasks.repackage_jetpack(data), None) def test_bad_addon_id(self): data = self.builder_data(addon=22) with self.assertRaises(models.ObjectDoesNotExist): tasks.repackage_jetpack(data) def test_bad_file_id(self): data = self.builder_data(file_id=234234) # Stick the file in the upgrader so it doesn't fail the uuid check. self.upgrader.file(234234, {'file': 234234, 'uuid': self.uuid}) with self.assertRaises(models.ObjectDoesNotExist): tasks.repackage_jetpack(data) def test_urllib_failure(self): self.urllib.urlretrieve.side_effect = StopIteration with self.assertRaises(StopIteration): tasks.repackage_jetpack(self.builder_data()) def test_new_file_hash(self): new_file = tasks.repackage_jetpack(self.builder_data()) hash_ = hashlib.sha256() hash_.update(open(self.xpi_path, 'rb').read()) eq_(new_file.hash, 'sha256:' + hash_.hexdigest()) def test_new_version(self): num_versions = self.addon.versions.count() new_file = tasks.repackage_jetpack(self.builder_data()) eq_(num_versions + 1, self.addon.versions.count()) new_version = self.addon.versions.latest() new_version.all_files = [new_file] def test_new_version_number(self): tasks.repackage_jetpack(self.builder_data()) new_version = self.addon.versions.latest() # From jetpack.xpi. eq_(new_version.version, '1.3') def test_new_file_status(self): new_file = tasks.repackage_jetpack(self.builder_data()) eq_(new_file.status, self.file.status) def test_addon_current_version(self): new_file = tasks.repackage_jetpack(self.builder_data()) addon = Addon.objects.get(id=self.addon.id) eq_(addon.current_version, new_file.version) def test_cancel(self): # Get the data so a row is set in redis. data = self.builder_data() self.upgrader.cancel() eq_(tasks.repackage_jetpack(data), None) def test_cancel_only_affects_bulk(self): # Get the data so a row is set in redis. data = self.builder_data() # Clear out everything. self.upgrader.cancel() # Put the file back in redis. self.upgrader.file(self.file.id, {'uuid': self.uuid, 'file': self.file.id, 'version': '1.0b4'}) # Try to clear again, this should skip the ownerless task above. self.upgrader.cancel() # The task should still work and return a new file. assert tasks.repackage_jetpack(data) def test_clear_redis_after_success(self): # Get the data so a row is set in redis. data = self.builder_data() assert tasks.repackage_jetpack(data) eq_(self.upgrader.file(self.file.id), {}) eq_(self.upgrader.version(), None) def test_email_sent(self): assert tasks.repackage_jetpack(self.builder_data()) eq_(len(mail.outbox), 1) def test_supported_apps(self): # Create AppVersions to match what's in the xpi. AppVersion.objects.create(application=amo.FIREFOX.id, version='3.6') AppVersion.objects.create(application=amo.FIREFOX.id, version='4.0b6') # Make sure the new appver matches the old appver. new = tasks.repackage_jetpack(self.builder_data()).version for old_app in self.file.version.apps.all(): new_app = new.apps.filter(application=old_app.application) eq_(new_app.values_list('min', 'max')[0], (old_app.min_id, old_app.max_id)) def test_block_duplicate_version(self): eq_(self.addon.versions.count(), 1) assert tasks.repackage_jetpack(self.builder_data()) eq_(self.addon.versions.count(), 2) # Make a new temp file for urlretrieve. tmp_file = self.create_temp_file() self.urllib.urlretrieve.return_value = (tmp_file, None) assert not tasks.repackage_jetpack(self.builder_data()) eq_(self.addon.versions.count(), 2) def test_file_on_mirror(self): # Make sure the mirror dir is clear. if storage.exists(os.path.dirname(self.file.mirror_file_path)): rmtree(os.path.dirname(self.file.mirror_file_path)) new_file = tasks.repackage_jetpack(self.builder_data()) assert storage.exists(new_file.mirror_file_path)
class TestRepackageJetpack(amo.tests.TestCase): fixtures = ['base/addon_3615'] def setUp(self): super(TestRepackageJetpack, self).setUp() urllib_patch = mock.patch('files.tasks.urllib') self.urllib = urllib_patch.start() self.addCleanup(urllib_patch.stop) self.addon = Addon.objects.get(id=3615) # Match the guid from jetpack.xpi. self.addon.update(guid='jid0-ODIKJS9b4IT3H1NYlPKr0NDtLuE@jetpack') self.file = self.addon.current_version.all_files[0] eq_(self.file.status, amo.STATUS_PUBLIC) # Set up a temp file so urllib.urlretrieve works. self.xpi_path = os.path.join(settings.ROOT, 'apps/files/fixtures/files/jetpack.xpi') tmp_file = tempfile.NamedTemporaryFile(delete=False) tmp_file.write(open(self.xpi_path, 'rb').read()) tmp_file.flush() self.urllib.urlretrieve.return_value = (tmp_file.name, None) self.upgrader = JetpackUpgrader() settings.SEND_REAL_EMAIL = True self.uuid = uuid.uuid4().hex def builder_data(self, **kw): """Generate builder_data response dictionary with sensible defaults.""" # Tell Redis we're sending to builder. self.upgrader.file(self.file.id, {'uuid': self.uuid, 'file': self.file.id, 'owner': 'bulk', 'version': '1.0b4'}) request = {'uuid': self.uuid, 'addon': self.addon.id, 'file_id': self.file.id} request.update((k, v) for k, v in kw.items() if k in request) data = {'result': 'success', 'msg': 'ok', 'location': 'http://new.file', # This fakes what we would send to the builder. 'request': urllib.urlencode(request)} data.update(kw) return data def test_result_not_success(self): data = self.builder_data(result='fail', msg='oops') eq_(tasks.repackage_jetpack(data), None) def test_bad_addon_id(self): data = self.builder_data(addon=22) with self.assertRaises(models.ObjectDoesNotExist): tasks.repackage_jetpack(data) def test_bad_file_id(self): data = self.builder_data(file_id=234234) # Stick the file in the upgrader so it doesn't fail the uuid check. self.upgrader.file(234234, {'file': 234234, 'uuid': self.uuid}) with self.assertRaises(models.ObjectDoesNotExist): tasks.repackage_jetpack(data) def test_urllib_failure(self): self.urllib.urlretrieve.side_effect = StopIteration with self.assertRaises(StopIteration): tasks.repackage_jetpack(self.builder_data()) def test_new_file_hash(self): new_file = tasks.repackage_jetpack(self.builder_data()) hash_ = hashlib.sha256() hash_.update(open(self.xpi_path, 'rb').read()) eq_(new_file.hash, 'sha256:' + hash_.hexdigest()) def test_new_version(self): num_versions = self.addon.versions.count() new_file = tasks.repackage_jetpack(self.builder_data()) eq_(num_versions + 1, self.addon.versions.count()) new_version = self.addon.versions.latest() new_version.all_files = [new_file] def test_new_version_number(self): tasks.repackage_jetpack(self.builder_data()) new_version = self.addon.versions.latest() # From jetpack.xpi. eq_(new_version.version, '1.3') def test_new_file_status(self): new_file = tasks.repackage_jetpack(self.builder_data()) eq_(new_file.status, self.file.status) def test_addon_current_version(self): new_file = tasks.repackage_jetpack(self.builder_data()) addon = Addon.objects.get(id=self.addon.id) eq_(addon.current_version, new_file.version) def test_cancel(self): # Get the data so a row is set in redis. data = self.builder_data() self.upgrader.cancel() eq_(tasks.repackage_jetpack(data), None) def test_cancel_only_affects_bulk(self): # Get the data so a row is set in redis. data = self.builder_data() # Clear out everything. self.upgrader.cancel() # Put the file back in redis. self.upgrader.file(self.file.id, {'uuid': self.uuid, 'file': self.file.id, 'version': '1.0b4'}) # Try to clear again, this should skip the ownerless task above. self.upgrader.cancel() # The task should still work and return a new file. assert tasks.repackage_jetpack(data) def test_clear_redis_after_success(self): # Get the data so a row is set in redis. data = self.builder_data() assert tasks.repackage_jetpack(data) eq_(self.upgrader.file(self.file.id), {}) eq_(self.upgrader.version(), None) def test_email_sent(self): assert tasks.repackage_jetpack(self.builder_data()) eq_(len(mail.outbox), 1) def test_supported_apps(self): version = tasks.repackage_jetpack(self.builder_data()).version old_apps = self.file.version.compatible_apps eq_(version.compatible_apps.keys(), old_apps.keys()) for app, appver in version.compatible_apps.items(): eq_(old_apps[app].max, appver.max) eq_(old_apps[app].min, appver.min)
def jetpack_resend(request, file_id): maxver = JetpackUpgrader().version() log.info('Starting a jetpack upgrade to %s [1 file].' % maxver) start_upgrade_task.delay([file_id], sdk_version=maxver) return redirect('zadmin.jetpack')
def setUp(self): super(TestCheckJetpackVersion, self).setUp() self.addon = Addon.objects.get(id=3615) JetpackUpgrader().jetpack_versions('1.0', '1.1')
class TestRepackageJetpack(amo.tests.TestCase): fixtures = ['base/addon_3615'] def setUp(self): super(TestRepackageJetpack, self).setUp() urllib_patch = mock.patch('files.tasks.urllib') self.urllib = urllib_patch.start() self.addCleanup(urllib_patch.stop) self.addon = Addon.objects.get(id=3615) # Match the guid from jetpack.xpi. self.addon.update(guid='jid0-ODIKJS9b4IT3H1NYlPKr0NDtLuE@jetpack') self.file = self.addon.current_version.all_files[0] eq_(self.file.status, amo.STATUS_PUBLIC) # Set up a temp file so urllib.urlretrieve works. self.xpi_path = os.path.join(settings.ROOT, 'apps/files/fixtures/files/jetpack.xpi') tmp_file = self.create_temp_file() self.urllib.urlretrieve.return_value = (tmp_file.name, None) self.upgrader = JetpackUpgrader() settings.SEND_REAL_EMAIL = True self.uuid = uuid.uuid4().hex def create_temp_file(self): tmp_file = tempfile.NamedTemporaryFile(delete=False, dir=settings.TMP_PATH) tmp_file.write(open(self.xpi_path, 'rb').read()) tmp_file.flush() return tmp_file def builder_data(self, **kw): """Generate builder_data response dictionary with sensible defaults.""" # Tell Redis we're sending to builder. self.upgrader.file( self.file.id, { 'uuid': self.uuid, 'file': self.file.id, 'owner': 'bulk', 'version': '1.0b4' }) request = { 'uuid': self.uuid, 'addon': self.addon.id, 'file_id': self.file.id } request.update((k, v) for k, v in kw.items() if k in request) data = { 'result': 'success', 'msg': 'ok', 'location': 'http://new.file', # This fakes what we would send to the builder. 'request': urllib.urlencode(request) } data.update(kw) return data def test_result_not_success(self): data = self.builder_data(result='fail', msg='oops') eq_(tasks.repackage_jetpack(data), None) def test_bad_addon_id(self): data = self.builder_data(addon=22) with self.assertRaises(models.ObjectDoesNotExist): tasks.repackage_jetpack(data) def test_bad_file_id(self): data = self.builder_data(file_id=234234) # Stick the file in the upgrader so it doesn't fail the uuid check. self.upgrader.file(234234, {'file': 234234, 'uuid': self.uuid}) with self.assertRaises(models.ObjectDoesNotExist): tasks.repackage_jetpack(data) def test_urllib_failure(self): self.urllib.urlretrieve.side_effect = StopIteration with self.assertRaises(StopIteration): tasks.repackage_jetpack(self.builder_data()) def test_new_file_hash(self): new_file = tasks.repackage_jetpack(self.builder_data()) hash_ = hashlib.sha256() hash_.update(open(self.xpi_path, 'rb').read()) eq_(new_file.hash, 'sha256:' + hash_.hexdigest()) def test_new_version(self): num_versions = self.addon.versions.count() new_file = tasks.repackage_jetpack(self.builder_data()) eq_(num_versions + 1, self.addon.versions.count()) new_version = self.addon.versions.latest() new_version.all_files = [new_file] def test_new_version_number(self): tasks.repackage_jetpack(self.builder_data()) new_version = self.addon.versions.latest() # From jetpack.xpi. eq_(new_version.version, '1.3') def test_new_file_status(self): new_file = tasks.repackage_jetpack(self.builder_data()) eq_(new_file.status, self.file.status) def test_addon_current_version(self): new_file = tasks.repackage_jetpack(self.builder_data()) addon = Addon.objects.get(id=self.addon.id) eq_(addon.current_version, new_file.version) def test_cancel(self): # Get the data so a row is set in redis. data = self.builder_data() self.upgrader.cancel() eq_(tasks.repackage_jetpack(data), None) def test_cancel_only_affects_bulk(self): # Get the data so a row is set in redis. data = self.builder_data() # Clear out everything. self.upgrader.cancel() # Put the file back in redis. self.upgrader.file(self.file.id, { 'uuid': self.uuid, 'file': self.file.id, 'version': '1.0b4' }) # Try to clear again, this should skip the ownerless task above. self.upgrader.cancel() # The task should still work and return a new file. assert tasks.repackage_jetpack(data) def test_clear_redis_after_success(self): # Get the data so a row is set in redis. data = self.builder_data() assert tasks.repackage_jetpack(data) eq_(self.upgrader.file(self.file.id), {}) eq_(self.upgrader.version(), None) def test_email_sent(self): assert tasks.repackage_jetpack(self.builder_data()) eq_(len(mail.outbox), 1) def test_supported_apps(self): # Create AppVersions to match what's in the xpi. AppVersion.objects.create(application_id=amo.FIREFOX.id, version='3.6') AppVersion.objects.create(application_id=amo.FIREFOX.id, version='4.0b6') # Make sure the new appver matches the old appver. new = tasks.repackage_jetpack(self.builder_data()).version for old_app in self.file.version.apps.all(): new_app = new.apps.filter(application=old_app.application) eq_( new_app.values_list('min', 'max')[0], (old_app.min_id, old_app.max_id)) def test_block_duplicate_version(self): eq_(self.addon.versions.count(), 1) assert tasks.repackage_jetpack(self.builder_data()) eq_(self.addon.versions.count(), 2) # Make a new temp file for urlretrieve. tmp_file = self.create_temp_file() self.urllib.urlretrieve.return_value = (tmp_file.name, None) assert not tasks.repackage_jetpack(self.builder_data()) eq_(self.addon.versions.count(), 2) def test_file_on_mirror(self): # Make sure the mirror dir is clear. shutil.rmtree(os.path.dirname(self.file.mirror_file_path)) new_file = tasks.repackage_jetpack(self.builder_data()) assert os.path.exists(new_file.mirror_file_path) def test_unreviewed_file_not_on_mirror(self): # Make sure the mirror dir is clear. shutil.rmtree(os.path.dirname(self.file.mirror_file_path)) self.file.update(status=amo.STATUS_UNREVIEWED) new_file = tasks.repackage_jetpack(self.builder_data()) assert not os.path.exists(new_file.mirror_file_path)