def inner_context(): stack = ExitStack() with stack: stack.enter_context(null_context()) stack.push(exit_cb) stack.callback(other_cb, 10, "hi", answer=42) yield
def test_instance_bypass(self): class Example(object): pass cm = Example() cm.__exit__ = object() stack = ExitStack() self.assertRaises(AttributeError, stack.enter_context, cm) stack.push(cm) self.assertIs(tuple(stack._exit_callbacks)[-1], cm)
def _exit(self, exc_type, exc_val, exc_tb, daemon=False): stack = ExitStack() # called last @stack.push def exit_loop(exc_type, exc_val, exc_tb): if self.client.is_closed: return self._loop.__exit__(exc_type, exc_val, exc_tb) if threading.current_thread() is threading.main_thread(): # TODO the main thread is not necessarily the last thread to finish. # Should the signal handler be removed in case it isn't? stack.push(self._signal_ctx) # called first # exit the client with the given daemon-ness, maybe leading the client to close @stack.push def exit_client(exc_type, exc_val, exc_tb): return self._call(self.client._aexit(exc_type, exc_val, exc_tb, daemon=daemon)) return stack.__exit__(exc_type, exc_val, exc_tb)
class ScienceBeamParserBaseSession: def __init__( self, parser: 'ScienceBeamParser', temp_dir: Optional[str] = None, fulltext_processor_config: Optional[FullTextProcessorConfig] = None, document_request_parameters: Optional[DocumentRequestParameters] = None ): self.parser = parser self.exit_stack = ExitStack() self._temp_dir: Optional[str] = temp_dir if fulltext_processor_config is None: fulltext_processor_config = parser.fulltext_processor_config self.fulltext_processor_config = fulltext_processor_config if document_request_parameters is None: document_request_parameters = DocumentRequestParameters() self.document_request_parameters = document_request_parameters def __enter__(self) -> 'ScienceBeamParserBaseSession': return self def close(self): self.exit_stack.close() def __exit__(self, exc, value, tb): self.close() @property def temp_dir(self) -> str: if not self._temp_dir: temp_dir_context = TemporaryDirectory( # pylint: disable=consider-using-with suffix='-sb-parser') self.exit_stack.push(temp_dir_context) self._temp_dir = temp_dir_context.__enter__() return self._temp_dir @property def temp_path(self) -> Path: return Path(self.temp_dir)
class Scope: """ A context manager that allows to register error and exit callbacks. """ _thread_locals = threading.local() @frozen class _ExitHandler: callback: Callable[[], Any] ignore_errors: bool = True def __exit__(self, exc_type, exc_value, exc_traceback): try: self.callback() except Exception: if not self.ignore_errors: raise @frozen class _ErrorHandler(_ExitHandler): def __exit__(self, exc_type, exc_value, exc_traceback): if exc_type: return super().__exit__(exc_type=exc_type, exc_value=exc_value, exc_traceback=exc_traceback) def __init__(self): self._stack = ExitStack() self.enabled = True def on_error_do(self, callback: Callable, *args, kwargs: Optional[Dict[str, Any]] = None, ignore_errors: bool = False): """ Registers a function to be called on scope exit because of an error. If ignore_errors is True, the errors from this function call will be ignored. """ self._register_callback(self._ErrorHandler, ignore_errors=ignore_errors, callback=callback, args=args, kwargs=kwargs) def on_exit_do(self, callback: Callable, *args, kwargs: Optional[Dict[str, Any]] = None, ignore_errors: bool = False): """ Registers a function to be called on scope exit. """ self._register_callback(self._ExitHandler, ignore_errors=ignore_errors, callback=callback, args=args, kwargs=kwargs) def _register_callback(self, handler_type, callback: Callable, args: Tuple[Any] = None, kwargs: Dict[str, Any] = None, ignore_errors: bool = False): if args or kwargs: callback = partial(callback, *args, **(kwargs or {})) self._stack.push(handler_type(callback, ignore_errors=ignore_errors)) def add(self, cm: ContextManager[T]) -> T: """ Enters a context manager and adds it to the exit stack. Returns: cm.__enter__() result """ return self._stack.enter_context(cm) def enable(self): self.enabled = True def disable(self): self.enabled = False def close(self): self.__exit__(None, None, None) def __enter__(self) -> Scope: return self def __exit__(self, exc_type, exc_value, exc_traceback): if not self.enabled: return self._stack.__exit__(exc_type, exc_value, exc_traceback) self._stack.pop_all() # prevent issues on repetitive calls @classmethod def current(cls) -> Scope: return cls._thread_locals.current @contextmanager def as_current(self): previous = getattr(self._thread_locals, 'current', None) self._thread_locals.current = self try: yield finally: self._thread_locals.current = previous
class TestKeyring(unittest.TestCase): """Test downloading and unpacking a keyring.""" @classmethod def setUpClass(self): SystemImagePlugin.controller.set_mode(cert_pem='cert.pem') def setUp(self): self._stack = ExitStack() try: self._serverdir = self._stack.enter_context(temporary_directory()) self._stack.push( make_http_server(self._serverdir, 8943, 'cert.pem', 'key.pem')) except: self._stack.close() raise def tearDown(self): self._stack.close() @configuration def test_good_path(self): # Everything checks out, with the simplest possible keyring.json. setup_keyrings('archive-master') setup_keyring_txz( 'spare.gpg', 'archive-master.gpg', dict(type='image-master'), os.path.join(self._serverdir, 'gpg', 'image-master.tar.xz')) get_keyring('image-master', 'gpg/image-master.tar.xz', 'archive-master') with Context(config.gpg.archive_master) as ctx: self.assertEqual(ctx.fingerprints, set(['289518ED3A0C4CFE975A0B32E0979A7EADE8E880'])) @configuration def test_good_path_full_json(self): # Everything checks out, with a fully loaded keyring.json file. next_year = datetime.now(tz=timezone.utc) + timedelta(days=365) setup_keyrings('archive-master') setup_keyring_txz( 'spare.gpg', 'archive-master.gpg', dict(type='image-master', expiry=next_year.timestamp(), model='nexus7'), os.path.join(self._serverdir, 'gpg', 'image-master.tar.xz')) get_keyring('image-master', 'gpg/image-master.tar.xz', 'archive-master') with Context(config.gpg.archive_master) as ctx: self.assertEqual(ctx.fingerprints, set(['289518ED3A0C4CFE975A0B32E0979A7EADE8E880'])) @configuration def test_good_path_model(self): # Everything checks out with the model specified. setup_keyrings() setup_keyring_txz( 'spare.gpg', 'archive-master.gpg', dict(type='image-master', model='nexus7'), os.path.join(self._serverdir, 'gpg', 'image-master.tar.xz')) get_keyring('image-master', 'gpg/image-master.tar.xz', 'archive-master') with Context(config.gpg.archive_master) as ctx: self.assertEqual(ctx.fingerprints, set(['289518ED3A0C4CFE975A0B32E0979A7EADE8E880'])) @configuration def test_good_path_expiry(self): # Everything checks out, with the expiration date specified. next_year = datetime.now(tz=timezone.utc) + timedelta(days=365) setup_keyrings('archive-master') setup_keyring_txz( 'spare.gpg', 'archive-master.gpg', dict(type='image-master', expiry=next_year.timestamp()), os.path.join(self._serverdir, 'gpg', 'image-master.tar.xz')) get_keyring('image-master', 'gpg/image-master.tar.xz', 'archive-master') with Context(config.gpg.archive_master) as ctx: self.assertEqual(ctx.fingerprints, set(['289518ED3A0C4CFE975A0B32E0979A7EADE8E880'])) @configuration def test_path_device_signing_keyring(self): # Get the device signing keyring. setup_keyrings('archive-master', 'image-master', 'image-signing') setup_keyring_txz( 'spare.gpg', 'image-signing.gpg', dict(type='device-signing'), os.path.join(self._serverdir, 'gpg', 'stable', 'nexus7', 'device-signing.tar.xz')) url = 'gpg/{}/{}/device-signing.tar.xz'.format(config.channel, config.device) get_keyring('device-signing', url, 'image-signing') with Context(config.gpg.device_signing) as ctx: self.assertEqual(ctx.fingerprints, set(['94BE2CECF8A5AF9F3A10E2A6526B7016C3D2FB44'])) @configuration def test_path_blacklist(self): # Get the blacklist keyring. setup_keyrings('archive-master', 'image-master') setup_keyring_txz( 'spare.gpg', 'image-master.gpg', dict(type='blacklist'), os.path.join(self._serverdir, 'gpg/blacklist.tar.xz')) url = 'gpg/blacklist.tar.xz'.format(config.channel, config.device) get_keyring('blacklist', url, 'image-master') blacklist_path = os.path.join(config.tempdir, 'blacklist.tar.xz') with Context(blacklist_path) as ctx: self.assertEqual(ctx.fingerprints, set(['94BE2CECF8A5AF9F3A10E2A6526B7016C3D2FB44'])) @configuration def test_tar_xz_file_missing(self): # If the tar.xz file cannot be downloaded, an error is raised. tarxz_path = os.path.join(self._serverdir, 'gpg', 'blacklist.tar.xz') setup_keyrings() setup_keyring_txz('spare.gpg', 'archive-master.gpg', dict(type='blacklist'), tarxz_path) os.remove(tarxz_path) self.assertRaises(FileNotFoundError, get_keyring, 'blacklist', 'gpg/blacklist.tar.xz', 'image-master') @configuration def test_asc_file_missing(self): # If the tar.xz.asc file cannot be downloaded, an error is raised. tarxz_path = os.path.join(self._serverdir, 'gpg', 'blacklist.tar.xz') setup_keyrings() setup_keyring_txz('spare.gpg', 'archive-master.gpg', dict(type='blacklist'), tarxz_path) os.remove(tarxz_path + '.asc') self.assertRaises(FileNotFoundError, get_keyring, 'blacklist', 'gpg/blacklist.tar.xz', 'image-master') @configuration def test_bad_signature(self): # Both files are downloaded, but the signature does not match the # image-master key. setup_keyrings() # Use the spare key as the blacklist, signed by itself. Since this # won't match the image-signing key, the check will fail. server_path = os.path.join(self._serverdir, 'gpg', 'blacklist.tar.xz') setup_keyring_txz('spare.gpg', 'spare.gpg', dict(type='blacklist'), server_path) with self.assertRaises(SignatureError) as cm: get_keyring('blacklist', 'gpg/blacklist.tar.xz', 'image-master') error = cm.exception # The local file name will be keyring.tar.xz in the cache directory. basename = os.path.basename self.assertEqual(basename(error.data_path), 'keyring.tar.xz') self.assertEqual(basename(error.signature_path), 'keyring.tar.xz.asc') # The crafted blacklist.tar.xz file will have an unpredictable # checksum due to tarfile variablility. with open(server_path, 'rb') as fp: checksum = hashlib.md5(fp.read()).hexdigest() self.assertEqual(error.data_checksum, checksum) # The signature file's checksum is also unpredictable. with open(server_path + '.asc', 'rb') as fp: checksum = hashlib.md5(fp.read()).hexdigest() self.assertEqual(error.signature_checksum, checksum) @configuration def test_blacklisted_signature(self): # Normally, the signature would be good, except that the fingerprint # of the device signing key is blacklisted. setup_keyrings('archive-master', 'image-master') blacklist = os.path.join(config.tempdir, 'gpg', 'blacklist.tar.xz') # Blacklist the image-master keyring. setup_keyring_txz('image-master.gpg', 'image-master.gpg', dict(type='blacklist'), blacklist) setup_keyring_txz( 'image-signing.gpg', 'image-master.gpg', dict(type='image-signing'), os.path.join(self._serverdir, 'gpg', 'image-signing.tar.xz')) # Now put an image-signing key on the server and attempt to download # it. Because the image-master is blacklisted, this will fail. self.assertRaises(SignatureError, get_keyring, 'image-signing', 'gpg/image-signing.tar.xz', 'image-master', blacklist) @configuration def test_bad_json_type(self): # This type, while the signatures match, the keyring type in the # keyring.json file does not match. setup_keyrings() setup_keyring_txz( 'device-signing.gpg', 'image-master.gpg', dict(type='master'), os.path.join(self._serverdir, 'gpg', 'blacklist.tar.xz')) with self.assertRaises(KeyringError) as cm: get_keyring('blacklist', 'gpg/blacklist.tar.xz', 'image-master') self.assertEqual( cm.exception.message, 'keyring type mismatch; wanted: blacklist, got: master') @configuration def test_bad_json_model(self): # Similar to above, but with a non-matching model name. setup_keyrings() setup_keyring_txz( 'device-signing.gpg', 'image-master.gpg', dict(type='blacklist', model='nexus0'), os.path.join(self._serverdir, 'gpg', 'blacklist.tar.xz')) with self.assertRaises(KeyringError) as cm: get_keyring('blacklist', 'gpg/blacklist.tar.xz', 'image-master') self.assertEqual( cm.exception.message, 'keyring model mismatch; wanted: nexus7, got: nexus0') @configuration def test_expired(self): # Similar to above, but the expiry key in the json names a utc # timestamp that has already elapsed. last_year = datetime.now(tz=timezone.utc) + timedelta(days=-365) setup_keyrings() setup_keyring_txz( 'device-signing.gpg', 'image-master.gpg', dict(type='blacklist', model='nexus7', expiry=last_year.timestamp()), os.path.join(self._serverdir, 'gpg', 'blacklist.tar.xz')) with self.assertRaises(KeyringError) as cm: get_keyring('blacklist', 'gpg/blacklist.tar.xz', 'image-master') self.assertEqual(cm.exception.message, 'expired keyring timestamp') @configuration def test_destination_image_master(self): # When a keyring is downloaded, we preserve its .tar.xz and # .tar.xz.asc files. setup_keyrings('archive-master') setup_keyring_txz( 'image-master.gpg', 'archive-master.gpg', dict(type='image-master'), os.path.join(self._serverdir, 'gpg', 'image-master.tar.xz')) asc_path = config.gpg.image_master + '.asc' self.assertFalse(os.path.exists(config.gpg.image_master)) self.assertFalse(os.path.exists(asc_path)) get_keyring('image-master', 'gpg/image-master.tar.xz', 'archive-master') self.assertTrue(os.path.exists(config.gpg.image_master)) self.assertTrue(os.path.exists(asc_path)) with Context(config.gpg.archive_master) as ctx: self.assertTrue(ctx.verify(asc_path, config.gpg.image_master)) @configuration def test_destination_image_signing(self): # When a keyring is downloaded, we preserve its .tar.xz and # .tar.xz.asc files. setup_keyrings('archive-master', 'image-master') setup_keyring_txz( 'image-signing.gpg', 'image-master.gpg', dict(type='image-signing'), os.path.join(self._serverdir, 'gpg', 'image-signing.tar.xz')) asc_path = config.gpg.image_signing + '.asc' self.assertFalse(os.path.exists(config.gpg.image_signing)) self.assertFalse(os.path.exists(asc_path)) get_keyring('image-signing', 'gpg/image-signing.tar.xz', 'image-master') self.assertTrue(os.path.exists(config.gpg.image_signing)) self.assertTrue(os.path.exists(asc_path)) with Context(config.gpg.image_master) as ctx: self.assertTrue(ctx.verify(asc_path, config.gpg.image_signing)) @configuration def test_destination_device_signing(self): # When a keyring is downloaded, we preserve its .tar.xz and # .tar.xz.asc files. setup_keyrings('archive-master', 'image-master', 'image-signing') setup_keyring_txz( 'device-signing.gpg', 'image-signing.gpg', dict(type='device-signing'), os.path.join(self._serverdir, 'stable', 'nexus7', 'device-signing.tar.xz')) asc_path = config.gpg.device_signing + '.asc' self.assertFalse(os.path.exists(config.gpg.device_signing)) self.assertFalse(os.path.exists(asc_path)) get_keyring('device-signing', 'stable/nexus7/device-signing.tar.xz', 'image-signing') self.assertTrue(os.path.exists(config.gpg.device_signing)) self.assertTrue(os.path.exists(asc_path)) with Context(config.gpg.image_signing) as ctx: self.assertTrue(ctx.verify(asc_path, config.gpg.device_signing)) @configuration def test_destination_blacklist(self): # Like above, but the blacklist files end up in the temporary # directory, since it's never persistent. setup_keyrings('archive-master', 'image-master') setup_keyring_txz( 'spare.gpg', 'image-master.gpg', dict(type='blacklist'), os.path.join(self._serverdir, 'gpg', 'blacklist.tar.xz')) txz_path = os.path.join(config.updater.data_partition, 'blacklist.tar.xz') asc_path = txz_path + '.asc' self.assertFalse(os.path.exists(txz_path)) self.assertFalse(os.path.exists(asc_path)) get_keyring('blacklist', 'gpg/blacklist.tar.xz', 'image-master') self.assertTrue(os.path.exists(txz_path)) self.assertTrue(os.path.exists(asc_path)) with Context(config.gpg.image_master) as ctx: self.assertTrue(ctx.verify(asc_path, txz_path))
class TestLoadChannel(unittest.TestCase): """Test downloading and caching the channels.json file.""" @classmethod def setUpClass(cls): SystemImagePlugin.controller.set_mode(cert_pem='cert.pem') def setUp(self): self._stack = ExitStack() self._state = State() try: self._serverdir = self._stack.enter_context(temporary_directory()) self._stack.push( make_http_server(self._serverdir, 8943, 'cert.pem', 'key.pem')) copy('channel.channels_01.json', self._serverdir, 'channels.json') self._channels_path = os.path.join(self._serverdir, 'channels.json') except: self._stack.close() raise def tearDown(self): self._stack.close() @configuration def test_load_channel_good_path(self): # A channels.json file signed by the image signing key, no blacklist. sign(self._channels_path, 'image-signing.gpg') setup_keyrings() self._state.run_thru('get_channel') channels = self._state.channels self.assertEqual(channels.daily.devices.nexus7.keyring.signature, '/daily/nexus7/device-keyring.tar.xz.asc') @configuration def test_load_channel_bad_signature(self): # We get an error if the signature on the channels.json file is bad. sign(self._channels_path, 'spare.gpg') setup_keyrings() self._state.run_thru('get_channel') # At this point, the state machine has determined that the # channels.json file is not signed with the cached image signing key, # so it will try to download a new imaging signing key. Let's put one # on the server, but it will not match the key that channels.json is # signed with. key_path = os.path.join(self._serverdir, 'gpg', 'image-signing.tar.xz') setup_keyring_txz('image-signing.gpg', 'image-master.gpg', dict(type='image-signing'), key_path) # This will succeed by grabbing a new image-signing key. from systemimage.testing.controller import stop_downloader stop_downloader(SystemImagePlugin.controller) next(self._state) # With the next state transition, we'll go back to trying to get the # channel.json file. Since the signature will still be bad, we'll get # a SignatureError this time. self.assertRaises(SignatureError, next, self._state) @configuration def test_load_channel_bad_signature_gets_fixed(self, config_d): # Like above, but the second download of the image signing key results # in a properly signed channels.json file. sign(self._channels_path, 'spare.gpg') setup_keyrings() self._state.run_thru('get_channel') # At this point, the state machine has determined that the # channels.json file is not signed with the cached image signing key, # so it will try to download a new imaging signing key. Let's put one # on the server, but it will not match the key that channels.json is # signed with. self.assertIsNone(self._state.channels) setup_keyring_txz( 'spare.gpg', 'image-master.gpg', dict(type='image-signing'), os.path.join(self._serverdir, 'gpg', 'image-signing.tar.xz')) # This will succeed by grabbing a new image-signing key. config = Configuration(config_d) with open(config.gpg.image_signing, 'rb') as fp: checksum = hashlib.md5(fp.read()).digest() next(self._state) with open(config.gpg.image_signing, 'rb') as fp: self.assertNotEqual(checksum, hashlib.md5(fp.read()).digest()) # The next state transition will find that the channels.json file is # properly signed. next(self._state) self.assertIsNotNone(self._state.channels) self.assertEqual( self._state.channels.daily.devices.nexus7.keyring.signature, '/daily/nexus7/device-keyring.tar.xz.asc') @configuration def test_load_channel_blacklisted_signature(self, config_d): # We get an error if the signature on the channels.json file is good # but the key is blacklisted. sign(self._channels_path, 'image-signing.gpg') setup_keyrings() setup_keyring_txz( 'image-signing.gpg', 'image-master.gpg', dict(type='blacklist'), os.path.join(self._serverdir, 'gpg', 'blacklist.tar.xz')) self._state.run_thru('get_channel') # We now have an image-signing key which is blacklisted. This will # cause the state machine to try to download a new image signing key, # so let's put the cached one up on the server. This will still be # backlisted though. config = Configuration(config_d) key_path = os.path.join(self._serverdir, 'gpg', 'image-signing.tar.xz') shutil.copy(config.gpg.image_signing, key_path) shutil.copy(config.gpg.image_signing + '.asc', key_path + '.asc') # Run the state machine through _get_channel() again, only this time # because the key is still blacklisted, we'll get an exception. self.assertRaises(SignatureError, self._state.run_thru, 'get_channel')
class TestDownloadIndex(unittest.TestCase): maxDiff = None @classmethod def setUpClass(self): SystemImagePlugin.controller.set_mode(cert_pem='cert.pem') def setUp(self): # Start the HTTPS server running. Vend it out of a temporary # directory which we load up with the right files. self._stack = ExitStack() try: self._serverdir = self._stack.enter_context(temporary_directory()) self._stack.push( make_http_server(self._serverdir, 8943, 'cert.pem', 'key.pem')) except: self._stack.close() raise def tearDown(self): self._stack.close() def _copysign(self, src, dst, keyring): server_dst = os.path.join(self._serverdir, dst) makedirs(os.path.dirname(server_dst)) copy(src, self._serverdir, dst) sign(server_dst, keyring) @configuration def test_load_index_good_path(self): # Load the index.json pointed to by the channels.json. All signatures # validate correctly and there is no device keyring or blacklist. self._copysign('index.channels_05.json', 'channels.json', 'image-signing.gpg') # index.index_04.json path B will win, with no bootme flags. self._copysign('index.index_04.json', 'stable/nexus7/index.json', 'image-signing.gpg') setup_keyrings() state = State() state.run_thru('get_index') self.assertEqual( state.index.global_.generated_at, datetime(2013, 4, 29, 18, 45, 27, tzinfo=timezone.utc)) self.assertEqual(state.index.images[0].files[1].checksum, 'bcd') @configuration def test_load_index_with_device_keyring(self): # Here, the index.json file is signed with a device keyring. self._copysign('index.channels_02.json', 'channels.json', 'image-signing.gpg') # index.index_04.json.json path B will win, with no bootme flags. self._copysign('index.index_04.json', 'stable/nexus7/index.json', 'device-signing.gpg') setup_keyrings() setup_keyring_txz( 'device-signing.gpg', 'image-signing.gpg', dict(type='device-signing'), os.path.join(self._serverdir, 'stable', 'nexus7', 'device.tar.xz')) state = State() state.run_thru('get_index') self.assertEqual( state.index.global_.generated_at, datetime(2013, 4, 29, 18, 45, 27, tzinfo=timezone.utc)) self.assertEqual(state.index.images[0].files[1].checksum, 'bcd') @configuration def test_load_index_with_device_keyring_and_signing_key(self): # Here, the index.json file is signed with the image signing keyring, # even though there is a device key. That's fine. self._copysign('index.channels_02.json', 'channels.json', 'image-signing.gpg') # index.index_04.json.json path B will win, with no bootme flags. self._copysign('index.index_04.json', 'stable/nexus7/index.json', 'image-signing.gpg') setup_keyrings() setup_keyring_txz( 'device-signing.gpg', 'image-signing.gpg', dict(type='device-signing'), os.path.join(self._serverdir, 'stable', 'nexus7', 'device.tar.xz')) state = State() state.run_thru('get_index') self.assertEqual( state.index.global_.generated_at, datetime(2013, 4, 29, 18, 45, 27, tzinfo=timezone.utc)) self.assertEqual(state.index.images[0].files[1].checksum, 'bcd') @configuration def test_load_index_with_bad_keyring(self): # Here, the index.json file is signed with a defective device keyring. self._copysign('index.channels_02.json', 'channels.json', 'image-signing.gpg') # This will be signed by a keyring that is not the device keyring. self._copysign('index.index_04.json', 'stable/nexus7/index.json', 'spare.gpg') setup_keyrings() setup_keyring_txz( 'device-signing.gpg', 'image-signing.gpg', dict(type='device-signing'), os.path.join(self._serverdir, 'stable', 'nexus7', 'device.tar.xz')) state = State() state.run_until('get_index') self.assertRaises(SignatureError, next, state) @configuration def test_load_index_with_blacklist(self): # Here, we've blacklisted the device key. self._copysign('index.channels_02.json', 'channels.json', 'image-signing.gpg') # This will be signed by a keyring that is not the device keyring. self._copysign('index.index_04.json', 'stable/nexus7/index.json', 'device-signing.gpg') setup_keyrings() setup_keyring_txz( 'device-signing.gpg', 'image-signing.gpg', dict(type='device-signing'), os.path.join(self._serverdir, 'stable', 'nexus7', 'device.tar.xz')) setup_keyring_txz( 'device-signing.gpg', 'image-master.gpg', dict(type='blacklist'), os.path.join(self._serverdir, 'gpg', 'blacklist.tar.xz')) state = State() state.run_until('get_index') self.assertRaises(SignatureError, next, state) @configuration def test_missing_channel(self): # The system's channel does not exist. self._copysign('index.channels_03.json', 'channels.json', 'image-signing.gpg') # index.index_04.json path B will win, with no bootme flags. self._copysign('index.index_04.json', 'stable/nexus7/index.json', 'image-signing.gpg') setup_keyrings() # Our channel (stable) isn't in the channels.json file, so there's # nothing to do. Running the state machine to its conclusion leaves # us with no index file. state = State() list(state) # There really is nothing left to do. self.assertIsNone(state.index) @configuration def test_missing_device(self): # The system's device does not exist. self._copysign('index.channels_04.json', 'channels.json', 'image-signing.gpg') # index.index_04.json path B will win, with no bootme flags. self._copysign('index.index_04.json', 'stable/nexus7/index.json', 'image-signing.gpg') setup_keyrings() # Our device (nexus7) isn't in the channels.json file, so there's # nothing to do. Running the state machine to its conclusion leaves # us with no index file. state = State() list(state) # There really is nothing left to do. self.assertIsNone(state.index)
class ServerTestBase(unittest.TestCase): # Must override in base classes. INDEX_FILE = None CHANNEL_FILE = None CHANNEL = None DEVICE = None SIGNING_KEY = 'device-signing.gpg' # For more detailed output. maxDiff = None @classmethod def setUpClass(self): # Avoid circular imports. from systemimage.testing.nose import SystemImagePlugin SystemImagePlugin.controller.set_mode(cert_pem='cert.pem') def setUp(self): # Avoid circular imports. from systemimage.state import State self._resources = ExitStack() self._state = State() try: self._serverdir = self._resources.enter_context( temporary_directory()) # Start up both an HTTPS and HTTP server. The data files are # vended over the latter, everything else, over the former. self._resources.push( make_http_server(self._serverdir, 8943, 'cert.pem', 'key.pem')) self._resources.push(make_http_server(self._serverdir, 8980)) # Set up the server files. assert self.CHANNEL_FILE is not None, ( 'Subclasses must set CHANNEL_FILE') copy(self.CHANNEL_FILE, self._serverdir, 'channels.json') sign(os.path.join(self._serverdir, 'channels.json'), 'image-signing.gpg') assert self.CHANNEL is not None, 'Subclasses must set CHANNEL' assert self.DEVICE is not None, 'Subclasses must set DEVICE' index_path = os.path.join(self._serverdir, self.CHANNEL, self.DEVICE, 'index.json') head, tail = os.path.split(index_path) assert self.INDEX_FILE is not None, ( 'Subclasses must set INDEX_FILE') copy(self.INDEX_FILE, head, tail) sign(index_path, self.SIGNING_KEY) setup_index(self.INDEX_FILE, self._serverdir, self.SIGNING_KEY) except: self._resources.close() raise self.addCleanup(self._resources.close) def _setup_server_keyrings(self, *, device_signing=True): # Only the archive-master key is pre-loaded. All the other keys # are downloaded and there will be both a blacklist and device # keyring. The four signed keyring tar.xz files and their # signatures end up in the proper location after the state machine # runs to completion. setup_keyrings('archive-master') setup_keyring_txz( 'spare.gpg', 'image-master.gpg', dict(type='blacklist'), os.path.join(self._serverdir, 'gpg', 'blacklist.tar.xz')) setup_keyring_txz( 'image-master.gpg', 'archive-master.gpg', dict(type='image-master'), os.path.join(self._serverdir, 'gpg', 'image-master.tar.xz')) setup_keyring_txz( 'image-signing.gpg', 'image-master.gpg', dict(type='image-signing'), os.path.join(self._serverdir, 'gpg', 'image-signing.tar.xz')) if device_signing: setup_keyring_txz( 'device-signing.gpg', 'image-signing.gpg', dict(type='device-signing'), os.path.join(self._serverdir, self.CHANNEL, self.DEVICE, 'device-signing.tar.xz'))
class TestDownload(unittest.TestCase): """Base class for testing the PyCURL and udm downloaders.""" def setUp(self): super().setUp() self._resources = ExitStack() try: # Start the HTTP server running, vending files out of our test # data directory. directory = os.path.dirname(data_path('__init__.py')) self._resources.push(make_http_server(directory, 8980)) except: self._resources.close() raise def tearDown(self): self._resources.close() super().tearDown() def _downloader(self, *args): return get_download_manager(*args) @configuration def test_good_path(self): # Download a bunch of files that exist. No callback. self._downloader().get_files(_http_pathify([ ('channel.channels_05.json', 'channels.json'), ('download.index_01.json', 'index.json'), ])) self.assertEqual( set(os.listdir(config.tempdir)), set(['channels.json', 'index.json'])) @configuration def test_empty_download(self): # Empty download set completes successfully. LP: #1245597. self._downloader().get_files([]) # No TimeoutError is raised. @configuration def test_user_agent(self): # The User-Agent request header contains the build number. version = random.randint(0, 99) config.build_number = version config.device = 'geddyboard' config.channel = 'devel-trio' # Download a magic path which the server will interpret to return us # the User-Agent header value. self._downloader().get_files(_http_pathify([ ('user-agent.txt', 'user-agent.txt'), ])) path = os.path.join(config.tempdir, 'user-agent.txt') with open(path, 'r', encoding='utf-8') as fp: user_agent = fp.read() self.assertEqual( user_agent, 'Ubuntu System Image Upgrade Client: ' 'device=geddyboard;channel=devel-trio;build={}'.format( version)) @configuration def test_download_with_callback(self): # Downloading calls the callback with some arguments. received_bytes = 0 total_bytes = 0 def callback(received, total): nonlocal received_bytes, total_bytes received_bytes = received total_bytes = total downloader = self._downloader(callback) downloader.get_files(_http_pathify([ ('channel.channels_05.json', 'channels.json'), ('download.index_01.json', 'index.json'), ])) self.assertEqual( set(os.listdir(config.tempdir)), set(['channels.json', 'index.json'])) self.assertEqual(received_bytes, 669) self.assertEqual(total_bytes, 669) @configuration def test_download_with_broken_callback(self): # If the callback raises an exception, it is logged and ignored. def callback(receive, total): raise RuntimeError exception = None def capture(message): nonlocal exception exception = message downloader = self._downloader(callback) with patch('systemimage.download.log.exception', capture): downloader.get_files(_http_pathify([ ('channel.channels_05.json', 'channels.json'), ])) # The exception got logged. self.assertEqual(exception, 'Exception in progress callback') # The file still got downloaded. self.assertEqual(os.listdir(config.tempdir), ['channels.json']) # This test helps bump the udm-based downloader test coverage to 100%. @unittest.skipIf(USING_PYCURL, 'Test is not relevant for PyCURL') @configuration def test_timeout(self): # If the reactor times out, we get an exception. We fake the timeout # by setting the attribute on the reactor, even though it successfully # completes its download without timing out. def finish_with_timeout(self, *args, **kws): self.timed_out = True self.quit() with patch('systemimage.udm.DownloadReactor._do_finished', finish_with_timeout): self.assertRaises( TimeoutError, self._downloader().get_files, _http_pathify([('channel.channels_05.json', 'channels.json')]) )
class TestCURL(unittest.TestCase): def setUp(self): super().setUp() self._resources = ExitStack() try: # Start the HTTP server running, vending files out of our test # data directory. directory = os.path.dirname(data_path('__init__.py')) self._resources.push(make_http_server(directory, 8980)) except: self._resources.close() raise def tearDown(self): self._resources.close() super().tearDown() @configuration def test_multi_perform(self): # PyCURL's multi.perform() can return the E_CALL_MULTI_PEFORM status # which tells us to just try again. This doesn't happen in practice, # but the code path needs coverage. However, .perform() itself can't # be mocked because pycurl.CurlMulti is a built-in. Fun. class FakeMulti: def perform(self): return pycurl.E_CALL_MULTI_PERFORM, 2 done_once = False class Testable(CurlDownloadManager): def _do_once(self, multi, handles): nonlocal done_once if done_once: return super()._do_once(multi, handles) else: done_once = True return super()._do_once(FakeMulti(), handles) Testable().get_files(_http_pathify([ ('channel.channels_05.json', 'channels.json'), ('download.index_01.json', 'index.json'), ])) self.assertTrue(done_once) # The files still get downloaded. self.assertEqual( set(os.listdir(config.tempdir)), set(['channels.json', 'index.json'])) @configuration def test_multi_fail(self): # PyCURL's multi.perform() can return a failure code (i.e. not E_OK) # which triggers a FileNotFoundError. It doesn't really matter which # failure code it returns. class FakeMulti: def perform(self): return pycurl.E_READ_ERROR, 2 class Testable(CurlDownloadManager): def _do_once(self, multi, handles): return super()._do_once(FakeMulti(), handles) with self.assertRaises(FileNotFoundError) as cm: Testable().get_files(_http_pathify([ ('channel.channels_05.json', 'channels.json'), ('download.index_01.json', 'index.json'), ])) # One of the two files will be contained in the error message, but # which one is undefined, although in practice it will be the first # one. self.assertRegex( cm.exception.args[0], 'http://localhost:8980/(channel.channels_05|index_01).json')
class TestDuplicateDownloads(unittest.TestCase): maxDiff = None def setUp(self): super().setUp() self._resources = ExitStack() try: self._serverdir = self._resources.enter_context( temporary_directory()) self._resources.push(make_http_server(self._serverdir, 8980)) except: self._resources.close() raise def tearDown(self): self._resources.close() super().tearDown() @configuration def test_matched_duplicates(self): # A download that duplicates the destination location, but for which # the sources and checksums are the same is okay. content = b'x' * 100 checksum = sha256(content).hexdigest() with open(os.path.join(self._serverdir, 'source.dat'), 'wb') as fp: fp.write(content) downloader = get_download_manager() downloads = [] for url, dst in _http_pathify([('source.dat', 'local.dat'), ('source.dat', 'local.dat'), ]): downloads.append(Record(url, dst, checksum)) downloader.get_files(downloads) self.assertEqual(os.listdir(config.tempdir), ['local.dat']) @configuration def test_mismatched_urls(self): # A download that duplicates the destination location, but for which # the source urls don't match, is not allowed. content = b'x' * 100 checksum = sha256(content).hexdigest() with open(os.path.join(self._serverdir, 'source1.dat'), 'wb') as fp: fp.write(content) with open(os.path.join(self._serverdir, 'source2.dat'), 'wb') as fp: fp.write(content) downloader = get_download_manager() downloads = [] for url, dst in _http_pathify([('source1.dat', 'local.dat'), ('source2.dat', 'local.dat'), ]): downloads.append(Record(url, dst, checksum)) with self.assertRaises(DuplicateDestinationError) as cm: downloader.get_files(downloads) self.assertEqual(len(cm.exception.duplicates), 1) dst, dupes = cm.exception.duplicates[0] self.assertEqual(os.path.basename(dst), 'local.dat') self.assertEqual([r[0] for r in dupes], ['http://localhost:8980/source1.dat', 'http://localhost:8980/source2.dat']) self.assertEqual(os.listdir(config.tempdir), []) @configuration def test_mismatched_checksums(self): # A download that duplicates the destination location, but for which # the checksums don't match, is not allowed. content = b'x' * 100 checksum = sha256(content).hexdigest() with open(os.path.join(self._serverdir, 'source.dat'), 'wb') as fp: fp.write(content) downloader = get_download_manager() url = urljoin(config.http_base, 'source.dat') downloads = [ Record(url, 'local.dat', checksum), # Mutate the checksum so they won't match. Record(url, 'local.dat', checksum[-1] + checksum[:-1]), ] with self.assertRaises(DuplicateDestinationError) as cm: downloader.get_files(downloads) self.assertEqual(len(cm.exception.duplicates), 1) dst, dupes = cm.exception.duplicates[0] self.assertEqual(os.path.basename(dst), 'local.dat') self.assertEqual([r[0] for r in dupes], ['http://localhost:8980/source.dat', 'http://localhost:8980/source.dat']) # The records in the exception aren't sorted by checksum. self.assertEqual( sorted(r[2] for r in dupes), ['09ecb6ebc8bcefc733f6f2ec44f791abeed6a99edf0cc31519637898aebd52d8' , '809ecb6ebc8bcefc733f6f2ec44f791abeed6a99edf0cc31519637898aebd52d' ]) self.assertEqual(os.listdir(config.tempdir), []) @configuration def test_duplicate_error_message(self): # When a duplicate destination error occurs, an error message gets # logged. Make sure the error message is helpful. content = b'x' * 100 checksum = sha256(content).hexdigest() with open(os.path.join(self._serverdir, 'source.dat'), 'wb') as fp: fp.write(content) downloader = get_download_manager() url = urljoin(config.http_base, 'source.dat') downloads = [ Record(url, 'local.dat', checksum), # Mutate the checksum so they won't match. Record(url, 'local.dat', checksum[-1] + checksum[:-1]), ] with self.assertRaises(DuplicateDestinationError) as cm: downloader.get_files(downloads) self.assertMultiLineEqual(str(cm.exception), """ [ ( 'local.dat', [ ( 'http://localhost:8980/source.dat', 'local.dat', '09ecb6ebc8bcefc733f6f2ec44f791abeed6a99edf0cc31519637898aebd52d8'), ( 'http://localhost:8980/source.dat', 'local.dat', '809ecb6ebc8bcefc733f6f2ec44f791abeed6a99edf0cc31519637898aebd52d')])]""")
class TestGSMDownloads(unittest.TestCase): def setUp(self): super().setUp() # Patch this method so that we can verify both the value of the flag # that system-image sets and the value that u-d-m's group downloader # records and uses. This is the only thing we can really # automatically test given that e.g. we won't have GSM in development. self._gsm_set_flag = None self._gsm_get_flag = None self._original = None def set_gsm(iface, *, allow_gsm): self._gsm_set_flag = allow_gsm self._original(iface, allow_gsm=allow_gsm) self._gsm_get_flag = iface.isGSMDownloadAllowed() self._resources = ExitStack() try: # Start the HTTP server running, vending files out of our test # data directory. directory = os.path.dirname(data_path('__init__.py')) self._resources.push(make_http_server(directory, 8980)) # Patch the GSM setting method to capture what actually happens. self._original = getattr(UDMDownloadManager, '_set_gsm') self._resources.enter_context(patch( 'systemimage.udm.UDMDownloadManager._set_gsm', set_gsm)) self._resources.callback(setattr, self, '_original', None) except: self._resources.close() raise def tearDown(self): self._resources.close() super().tearDown() @configuration def test_manual_downloads_gsm_allowed(self, config_d): # When auto_download is 0, manual downloads are enabled so assuming # the user knows what they're doing, GSM downloads are allowed. config = Configuration(config_d) Settings(config).set('auto_download', '0') get_download_manager().get_files(_http_pathify([ ('channel.channels_05.json', 'channels.json') ])) self.assertTrue(self._gsm_set_flag) self.assertTrue(self._gsm_get_flag) @configuration def test_wifi_downloads_gsm_disallowed(self, config_d): # Obviously GSM downloads are not allowed when downloading # automatically on wifi-only. config = Configuration(config_d) Settings(config).set('auto_download', '1') get_download_manager().get_files(_http_pathify([ ('channel.channels_05.json', 'channels.json') ])) self.assertFalse(self._gsm_set_flag) self.assertFalse(self._gsm_get_flag) @configuration def test_always_downloads_gsm_allowed(self, config_d): # GSM downloads are allowed when always downloading. config = Configuration(config_d) Settings(config).set('auto_download', '2') get_download_manager().get_files(_http_pathify([ ('channel.channels_05.json', 'channels.json') ])) self.assertTrue(self._gsm_set_flag) self.assertTrue(self._gsm_get_flag)
def backup(repo, progress=None, single=False): """Perform a backup This is usually called from Repository.backup() and is tightly integrated with the Repository class. It lives in its own module for organizational reasons. :type repo: backathon.repository.Repository :param progress: A callback function that provides status updates on the scan :param single: If this parameter is true, the backup process will all happen in a single thread. This can help with debugging and profiling. The progress callable takes two parameters: the backup count and backup total. """ if models.FSEntry.objects.using(repo.db).filter(new=True).exists(): # This happens when a new root is added but hasn't been scanned yet. raise RuntimeError("You need to run a scan first") to_backup = models.FSEntry.objects.using(repo.db).filter(obj__isnull=True) # The ready_to_backup set is the set of all nodes whose children have all # already been backed up. In other words, these are the entries that we # can back up right now. ready_to_backup = to_backup.exclude( # The sub query selects the *parents* of entries that are not yet # backed up. Therefore, we're excluding entries whose children are # not yet backed up. id__in=to_backup.exclude(parent__isnull=True).values("parent_id")) # The two above querysets remain unevaluated. We therefore get new results # on each call to .exists() below. Calls to .iterator() always return new # results. backup_total = to_backup.count() backup_count = 0 if single: executor = DummyExecutor() else: executor = concurrent.futures.ProcessPoolExecutor( max_workers=NUM_WORKERS, ) # SQLite connections should not be forked, according to the SQLite # documentation. Django and/or Python may have some protections # from this problem, but I'm not aware of any, so I'm taking caution and # closing all connections before forcing the process pool to immediately # launch the processes by submitting a dummy task. connections.close_all() executor.submit(time.time).result() tasks = set() contexts = ExitStack() with contexts: contexts.enter_context(executor) # Cancel all tasks that haven't been started yet def on_exit(): for t in tasks: t.cancel() contexts.callback(on_exit) def catch_sigint(exc_type, exc_value, traceback): if exc_type and issubclass(exc_type, KeyboardInterrupt): print() print("Ctrl-C caught. Finishing the current batch of " "uploads, please wait...") contexts.push(catch_sigint) while to_backup.exists(): ct = 0 last_checkpoint = time.monotonic() iterator = ready_to_backup.iterator() for entry_batch in batcher(iterator, BATCH_SIZE): ct += 1 # Assert our query is working correctly and that there are no # SQLite isolation problems (entries we've already backed up # re-appearing later in the same query) assert all(entry.obj_id is None for entry in entry_batch) tasks.add(executor.submit(backup_entry, repo, entry_batch)) # Don't put the entire to_backup result set in the queue at # once, to save memory. # If there are too many unfinished tasks, wait for one to # finish. if len(tasks) >= NUM_WORKERS + 1 or single: done, tasks = concurrent.futures.wait( tasks, timeout=None, return_when=concurrent.futures.FIRST_COMPLETED, ) for f in done: backup_count += f.result() if progress is not None: progress(backup_count, backup_total) # SQLite won't auto-checkpoint the write-ahead log while we # have the query iterator still open. So we force the inner # loop to exit every once in a while and force a WAL # checkpoint to keep the WAL from growing unbounded. if time.monotonic() - last_checkpoint > 30: # Note: closing the iterator should close the cursor # within it, but I think this is relying on reference # counted garbage collection. # If we run into problems, we'll have to find a different # strategy to run checkpoints iterator.close() with connections[repo.db].cursor() as cursor: cursor.execute("PRAGMA wal_checkpoint=RESTART") # Sanity check: if we entered the outer loop but the inner loop's # query didn't select anything, then we're not making progress and # may be caught in an infinite loop. In particular, this could happen # if we somehow got a cycle in the FSEntry tree in the database. # There would be entries needing backing up, but none of them have # all their dependent children backed up. assert ct > 0 # Collect results for the rest of the tasks. We have to do this # at the end of each inner loop to guarantee a correct ordering # to backed up entries. Items selected next loop could depend on # items still in process in the pool. # This stalls the workers but it doesn't end up costing all that # much time compared to time spent working. for f in concurrent.futures.as_completed(tasks): backup_count += f.result() if progress is not None: progress(backup_count, backup_total) tasks.clear() # End of outer "while" loop, and end of the contexts ExitStack. The # Executor is shut down at this point. # Now add the Snapshot object(s) to the database representing this backup # run. There's one snapshot per root, but we give them all the same datetime # so they can still be grouped together in queries. now = timezone.now() for root in models.FSEntry.objects.using( repo.db).filter(parent__isnull=True): assert root.obj_id is not None with atomic_immediate(using=repo.db): ss = models.Snapshot.objects.using(repo.db).create( path=root.path, root_id=root.obj_id, date=now, ) repo.put_snapshot(ss) with connections[repo.db].cursor() as cursor: cursor.execute("ANALYZE")
class TestWinnerDownloads(unittest.TestCase): """Test full end-to-end downloads through index.json.""" @classmethod def setUpClass(self): SystemImagePlugin.controller.set_mode(cert_pem='cert.pem') def setUp(self): # Start both an HTTP and an HTTPS server running. The former is for # the zip files and the latter is for everything else. Vend them out # of a temporary directory which we load up with the right files. self._stack = ExitStack() try: self._serverdir = self._stack.enter_context(temporary_directory()) copy('winner.channels_01.json', self._serverdir, 'channels.json') sign(os.path.join(self._serverdir, 'channels.json'), 'image-signing.gpg') # Path B will win, with no bootme flags. self._indexpath = os.path.join('stable', 'nexus7', 'index.json') copy('winner.index_02.json', self._serverdir, self._indexpath) sign(os.path.join(self._serverdir, self._indexpath), 'image-signing.gpg') # Create every file in path B. The file contents will be the # checksum value. We need to create the signatures on the fly. setup_index('winner.index_02.json', self._serverdir, 'image-signing.gpg') self._stack.push( make_http_server(self._serverdir, 8943, 'cert.pem', 'key.pem')) self._stack.push(make_http_server(self._serverdir, 8980)) except: self._stack.close() raise def tearDown(self): self._stack.close() @configuration def test_calculate_candidates(self): # Calculate the candidate paths. setup_keyrings() state = State() # Run the state machine until we get an index file. state.run_until('calculate_winner') candidates = get_candidates(state.index, 100) # There are three candidate upgrade paths. self.assertEqual(len(candidates), 3) descriptions = [] for image in candidates[0]: # There's only one description per image so order doesn't matter. descriptions.extend(image.descriptions.values()) self.assertEqual(descriptions, ['Full A', 'Delta A.1', 'Delta A.2']) descriptions = [] for image in candidates[1]: # There's only one description per image so order doesn't matter. descriptions.extend(image.descriptions.values()) self.assertEqual(descriptions, ['Full B', 'Delta B.1', 'Delta B.2']) descriptions = [] for image in candidates[2]: # There's only one description per image so order doesn't matter. descriptions.extend(image.descriptions.values()) self.assertEqual(descriptions, ['Full C', 'Delta C.1']) @configuration def test_calculate_winner(self): # Calculate the winning upgrade path. setup_keyrings() state = State() touch_build(100) # Run the state machine long enough to get the candidates and winner. state.run_thru('calculate_winner') # There are three candidate upgrade paths. descriptions = [] for image in state.winner: # There's only one description per image so order doesn't matter. descriptions.extend(image.descriptions.values()) self.assertEqual(descriptions, ['Full B', 'Delta B.1', 'Delta B.2']) @configuration def test_download_winners(self): # Check that all the winning path's files are downloaded. setup_keyrings() state = State() touch_build(100) # Run the state machine until we download the files. state.run_thru('download_files') # The B path files contain their checksums. def assert_file_contains(filename, contents): path = os.path.join(config.updater.cache_partition, filename) with open(path, encoding='utf-8') as fp: self.assertEqual(fp.read(), contents) assert_file_contains('5.txt', '345') assert_file_contains('6.txt', '456') assert_file_contains('7.txt', '567') # Delta B.1 files. assert_file_contains('8.txt', '678') assert_file_contains('9.txt', '789') assert_file_contains('a.txt', '89a') # Delta B.2 files. assert_file_contains('b.txt', '9ab') assert_file_contains('d.txt', 'fed') assert_file_contains('c.txt', 'edc') @configuration def test_download_winners_overwrite(self): # Check that all the winning path's files are downloaded, even if # those files already exist in their destination paths. setup_keyrings() state = State() touch_build(100) # Run the state machine until we download the files. for basename in '56789abcd': base = os.path.join(config.updater.cache_partition, basename) path = base + '.txt' with open(path, 'w', encoding='utf-8') as fp: print('stale', file=fp) state.run_thru('download_files') # The B path files contain their checksums. def assert_file_contains(filename, contents): path = os.path.join(config.updater.cache_partition, filename) with open(path, encoding='utf-8') as fp: self.assertEqual(fp.read(), contents) assert_file_contains('5.txt', '345') assert_file_contains('6.txt', '456') assert_file_contains('7.txt', '567') # Delta B.1 files. assert_file_contains('8.txt', '678') assert_file_contains('9.txt', '789') assert_file_contains('a.txt', '89a') # Delta B.2 files. assert_file_contains('b.txt', '9ab') assert_file_contains('d.txt', 'fed') assert_file_contains('c.txt', 'edc') @configuration def test_download_winners_signed_by_device_key(self): # Check that all the winning path's files are downloaded, even when # they are signed by the device key instead of the image signing # master. setup_keyrings() # To set up the device signing key, we need to load channels_03.json # and copy the device keyring to the server. copy('winner.channels_02.json', self._serverdir, 'channels.json') sign(os.path.join(self._serverdir, 'channels.json'), 'image-signing.gpg') setup_keyring_txz( 'device-signing.gpg', 'image-signing.gpg', dict(type='device-signing'), os.path.join(self._serverdir, 'stable', 'nexus7', 'device.tar.xz')) # The index.json file and all the downloadable files must now be # signed with the device key. sign(os.path.join(self._serverdir, self._indexpath), 'device-signing.gpg') setup_index('winner.index_02.json', self._serverdir, 'device-signing.gpg') touch_build(100) # Run the state machine until we download the files. state = State() state.run_thru('download_files') # The B path files contain their checksums. def assert_file_contains(filename, contents): path = os.path.join(config.updater.cache_partition, filename) with open(path, encoding='utf-8') as fp: self.assertEqual(fp.read(), contents) assert_file_contains('5.txt', '345') assert_file_contains('6.txt', '456') assert_file_contains('7.txt', '567') # Delta B.1 files. assert_file_contains('8.txt', '678') assert_file_contains('9.txt', '789') assert_file_contains('a.txt', '89a') # Delta B.2 files. assert_file_contains('b.txt', '9ab') assert_file_contains('d.txt', 'fed') assert_file_contains('c.txt', 'edc') @configuration def test_download_winners_signed_by_signing_key_with_device_key(self): # Check that all the winning path's files are downloaded, even when # they are signed by the device key instead of the image signing # master. setup_keyrings() # To set up the device signing key, we need to load this channels.json # file and copy the device keyring to the server. copy('winner.channels_02.json', self._serverdir, 'channels.json') sign(os.path.join(self._serverdir, 'channels.json'), 'image-signing.gpg') setup_keyring_txz( 'device-signing.gpg', 'image-signing.gpg', dict(type='device-signing'), os.path.join(self._serverdir, 'stable', 'nexus7', 'device.tar.xz')) sign(os.path.join(self._serverdir, self._indexpath), 'device-signing.gpg') # All the downloadable files are now signed with the image signing key. setup_index('winner.index_02.json', self._serverdir, 'image-signing.gpg') touch_build(100) # Run the state machine until we download the files. state = State() state.run_thru('download_files') # The B path files contain their checksums. def assert_file_contains(filename, contents): path = os.path.join(config.updater.cache_partition, filename) with open(path, encoding='utf-8') as fp: self.assertEqual(fp.read(), contents) assert_file_contains('5.txt', '345') assert_file_contains('6.txt', '456') assert_file_contains('7.txt', '567') # Delta B.1 files. assert_file_contains('8.txt', '678') assert_file_contains('9.txt', '789') assert_file_contains('a.txt', '89a') # Delta B.2 files. assert_file_contains('b.txt', '9ab') assert_file_contains('d.txt', 'fed') assert_file_contains('c.txt', 'edc') @configuration def test_download_winners_bad_checksums(self): # Similar to the various good paths, except because the checksums are # wrong in this index.json file, we'll get a error when downloading. copy('winner.index_01.json', self._serverdir, self._indexpath) sign(os.path.join(self._serverdir, self._indexpath), 'image-signing.gpg') setup_index('winner.index_01.json', self._serverdir, 'image-signing.gpg') setup_keyrings() state = State() touch_build(100) # Run the state machine until we're prepped to download state.run_until('download_files') # Now try to download the files and get the error. with self.assertRaises(FileNotFoundError) as cm: next(state) self.assertIn('HASH ERROR', str(cm.exception)) @configuration def test_download_winners_signed_by_wrong_key(self): # There is a device key, but the image files are signed by the image # signing key, which according to the spec means the files are not # signed correctly. setup_keyrings() # To set up the device signing key, we need to load this channels.json # file and copy the device keyring to the server. copy('winner.channels_02.json', self._serverdir, 'channels.json') sign(os.path.join(self._serverdir, 'channels.json'), 'image-signing.gpg') setup_keyring_txz( 'device-signing.gpg', 'image-signing.gpg', dict(type='device-signing'), os.path.join(self._serverdir, 'stable', 'nexus7', 'device.tar.xz')) sign(os.path.join(self._serverdir, self._indexpath), 'device-signing.gpg') # All the downloadable files are now signed with a bogus key. setup_index('winner.index_02.json', self._serverdir, 'spare.gpg') touch_build(100) # Run the state machine until just before we download the files. state = State() state.run_until('download_files') # The next state transition will fail because of the missing signature. self.assertRaises(SignatureError, next, state) # There are no downloaded files. txtfiles = set(filename for filename in os.listdir(config.tempdir) if os.path.splitext(filename)[1] == '.txt') self.assertEqual(len(txtfiles), 0) @configuration def test_no_download_winners_with_missing_signature(self): # If one of the download files is missing a signature, none of the # files get downloaded and get_files() fails. setup_keyrings() state = State() touch_build(100) # Remove a signature. os.remove(os.path.join(self._serverdir, '6/7/8.txt.asc')) # Run the state machine to calculate the winning path. state.run_until('download_files') # The next state transition will fail because of the missing signature. self.assertRaises(FileNotFoundError, next, state) # There are no downloaded files. txtfiles = set(filename for filename in os.listdir(config.tempdir) if os.path.splitext(filename)[1] == '.txt') self.assertEqual(len(txtfiles), 0, txtfiles) @configuration def test_no_download_winners_with_bad_signature(self): # If one of the download files has a bad a signature, none of the # downloaded files are available. setup_keyrings() state = State() touch_build(100) # Break a signature sign(os.path.join(self._serverdir, '6', '7', '8.txt'), 'spare.gpg') # Run the state machine to calculate the winning path. state.run_until('download_files') # The next state transition will fail because of the missing signature. self.assertRaises(SignatureError, next, state) # There are no downloaded files. txtfiles = set(filename for filename in os.listdir(config.tempdir) if os.path.splitext(filename)[1] == '.txt') self.assertEqual(len(txtfiles), 0)