def main(): catalogs = { 'dcp2': config.Catalog(name='dcp2', atlas='hca', internal=False, plugins=dict( metadata=config.Catalog.Plugin(name='hca'), repository=config.Catalog.Plugin(name='tdr')), sources=set()) } # To create a normalized OpenAPI document, we patch any # deployment-specific variables that affect the document. with patch.object(target=type(config), attribute='catalogs', new_callable=PropertyMock, return_value=catalogs): assert config.catalogs == catalogs with patch.object(target=config, attribute='service_function_name', return_value='azul_service'): assert config.service_name == 'azul_service' with patch.object(target=config, attribute='service_endpoint', return_value='localhost'): assert config.service_endpoint() == 'localhost' app_module = load_app_module('service') app_spec = app_module.app.spec() doc_path = os.path.join(config.project_root, 'lambdas/service/openapi.json') with write_file_atomically(doc_path) as file: json.dump(app_spec, file, indent=4)
def test_cached_health(self): self.storage_service.create_bucket() # No health object is available in S3 bucket, yielding an error with self.helper() as helper: response = requests.get( str(self.base_url.set(path='/health/cached'))) self.assertEqual(500, response.status_code) self.assertEqual( 'ChaliceViewError: Cached health object does not exist', response.json()['Message']) # A successful response is obtained when all the systems are functional self._create_mock_queues() endpoint_states = self._endpoint_states() app = load_app_module(self.lambda_name(), unit_test=True) with self.helper() as helper: self._mock_service_endpoints(helper, endpoint_states) app.update_health_cache(MagicMock(), MagicMock()) response = requests.get( str(self.base_url.set(path='/health/cached'))) self.assertEqual(200, response.status_code) # Another failure is observed when the cache health object is older than 2 minutes future_time = time.time() + 3 * 60 with self.helper() as helper: with patch('time.time', new=lambda: future_time): response = requests.get( str(self.base_url.set(path='/health/cached'))) self.assertEqual(500, response.status_code) self.assertEqual( 'ChaliceViewError: Cached health object is stale', response.json()['Message'])
def load_tests(_loader, tests, _ignore): root = azul.config.project_root for module in [ azul, azul.azulclient, azul.collections, azul.doctests, azul.dss, azul.exceptions, azul.files, azul.indexer.aggregate, azul.json, azul.json_freeze, azul.objects, azul.openapi, azul.openapi.params, azul.openapi.responses, azul.openapi.schema, azul.plugins.metadata.hca.contributor_matrices, azul.plugins.metadata.hca.full_metadata, azul.plugins.repository.tdr, azul.plugins.metadata.hca.transform, azul.service.drs_controller, azul.service.manifest_service, azul.strings, azul.terra, azul.threads, azul.time, azul.types, azul.uuids, azul.vendored.frozendict, retorts, load_app_module('service'), load_module(root + '/scripts/envhook.py', 'envhook'), load_module(root + '/scripts/export_environment.py', 'export_environment'), load_module(root + '/scripts/check_branch.py', 'check_branch'), load_module(root + '/scripts/velocity.py', 'velocity'), load_module(root + '/.flake8/azul_flake8.py', 'azul_flake8') ]: suite = doctest.DocTestSuite(module) assert suite.countTestCases() > 0, module tests.addTests(suite) return tests
def manage_lambdas(self, queues: Mapping[str, Queue], enable: bool): """ Enable or disable the readers and writers of the given queues """ indexer = load_app_module('indexer', unit_test=True) functions_by_queue = { handler.queue: config.indexer_function_name(handler.name) for handler in indexer.app.handler_map.values() if hasattr(handler, 'queue') } with ThreadPoolExecutor(max_workers=len(queues)) as tpe: futures = [] def submit(f, *args, **kwargs): futures.append(tpe.submit(f, *args, **kwargs)) for queue_name, queue in queues.items(): try: function = functions_by_queue[queue_name] except KeyError: assert queue_name in config.fail_queue_names else: if queue_name == config.notifications_queue_name(): # Prevent new notifications from being added submit(self._manage_lambda, config.indexer_name, enable) submit(self._manage_sqs_push, function, queue, enable) self._handle_futures(futures) futures = [tpe.submit(self._wait_for_queue_idle, queue) for queue in queues.values()] self._handle_futures(futures)
def setUpClass(cls): super().setUpClass() # Load the application module without modifying `sys.path` and without # adding it to `sys.modules`. This simplifies tear down and isolates the # app modules from different lambdas loaded by different concrete # subclasses. It does, however, violate this one invariant: # `sys.modules[module.__name__] == module` cls.app_module = load_app_module(cls.lambda_name(), unit_test=True)
def setUpClass(cls) -> None: super().setUpClass() app_module = load_app_module('service') app_dir = os.path.dirname(app_module.__file__) factory = chalice.cli.factory.CLIFactory(app_dir) config = factory.create_config_obj() cls.server = factory.create_local_server(app_obj=app_module.app, config=config, host=cls.url.host, port=cls.url.port) cls.server_thread = threading.Thread(target=cls.server.serve_forever) cls.server_thread.start()
def load_tests(_loader, tests, _ignore): root = azul.config.project_root for module in [ azul, azul.auth, azul.azulclient, azul.bigquery, azul.caching, azul.collections, azul.doctests, azul.dss, azul.exceptions, azul.files, azul.indexer, azul.indexer.aggregate, azul.iterators, azul.json, azul.json_freeze, azul.objects, azul.openapi, azul.openapi.params, azul.openapi.responses, azul.openapi.schema, azul.plugins.metadata.hca.contributor_matrices, azul.plugins.repository.tdr, azul.plugins.metadata.hca.transform, azul.service.drs_controller, azul.service.manifest_service, azul.service.repository_controller, azul.strings, azul.terra, azul.terraform, azul.threads, azul.time, azul.types, azul.uuids, azul.vendored.frozendict, load_app_module('service', unit_test=True), load_script('check_branch'), load_script('envhook'), load_script('export_environment'), load_module(root + '/.flake8/azul_flake8.py', 'azul_flake8'), load_module(root + '/test/test_tagging.py', 'test_tagging'), load_module(root + '/test/indexer/test_tdr.py', 'test_tdr') ]: suite = doctest.DocTestSuite(module) assert suite.countTestCases() > 0, module tests.addTests(suite) return tests
def _contribution_lambda_names(cls) -> FrozenSet[str]: indexer = load_app_module('indexer') notification_queue_names = { config.unqual_notifications_queue_name(retry=retry) for retry in (False, True) } def has_notification_queue(handler) -> bool: try: queue = handler.queue except AttributeError: return False else: resource_name, _ = config.unqualified_resource_name(queue) return resource_name in notification_queue_names return frozenset((handler.name for handler in vars(indexer).values() if has_notification_queue(handler)))
def main(): static_dir = os.path.join(parent_dir, 'apidev_static') web_page = os.path.join(static_dir, 'swagger-editor.html') host, port = 'localhost', 8787 server_url = f"http://{host}:{port}" httpd = HTTPServer((host, port), SimpleHTTPRequestHandler) address = f"{server_url}/{os.path.relpath(web_page)}?url={server_url}/{os.path.relpath(parent_dir)}/{spec_file}" print(f'Open {address} in browser to validate changes.') service = load_app_module('service') event_handler = UpdateHandler(service) write_specs(service.app) observer = Observer() observer.schedule(event_handler, path=os.path.dirname(service.__file__), recursive=False) observer.start() httpd.serve_forever()
from azul import ( config, ) from azul.modules import ( load_app_module, ) from azul.template import ( emit, ) suffix = '-' + config.deployment_stage assert config.indexer_name.endswith(suffix) indexer = load_app_module('indexer') emit({ "version": "2.0", "app_name": config. indexer_name[:-len(suffix)], # Chalice appends stage name implicitly "api_gateway_stage": config.deployment_stage, "manage_iam_role": False, "iam_role_arn": "${var.role_arn}", "environment_variables": config.lambda_env, "minimum_compression_size": config.minimum_compression_size, "lambda_timeout": config.api_gateway_lambda_timeout, "lambda_memory_size": 128, "stages": { config.deployment_stage: { "lambda_functions": { indexer.contribute.name: { "reserved_concurrency": config.contribution_concurrency(retry=False), "lambda_memory_size": 256,
def test(self, mock_uuid, mock_helper): service = load_app_module('service', unit_test=True) # In a LocalAppTestCase we need the actual state machine name state_machine_name = config.state_machine_name(service.generate_manifest.name) with responses.RequestsMock() as helper: helper.add_passthru(str(self.base_url)) for fetch in (True, False): with self.subTest(fetch=fetch): execution_id = '6c9dfa3f-e92e-11e8-9764-ada973595c11' mock_uuid.return_value = execution_id format_ = ManifestFormat.compact filters = Filters(explicit={'organ': {'is': ['lymph node']}}, source_ids={'6aaf72a6-0a45-5886-80cf-48f8d670dc26'}) params = { 'catalog': self.catalog, 'format': format_.value, 'filters': json.dumps(filters.reify(service_config=MagicMock(), explicit_only=True)) } path = '/manifest/files' object_url = 'https://url.to.manifest?foo=bar' file_name = 'some_file_name' object_key = f'manifests/{file_name}' manifest_url = self.base_url.set(path=path, args=dict(params, objectKey=object_key)) manifest = Manifest(location=object_url, was_cached=False, format_=format_, catalog=self.catalog, filters=filters, object_key=object_key, file_name=file_name) url = self.base_url.set(path=path, args=params) if fetch: url.path.segments.insert(0, 'fetch') partitions = ( ManifestPartition(index=0, is_last=False, file_name=None, config=None, multipart_upload_id=None, part_etags=None, page_index=None, is_last_page=None, search_after=None), ManifestPartition(index=1, is_last=False, file_name=file_name, config={}, multipart_upload_id='some_upload_id', part_etags=('some_etag',), page_index=512, is_last_page=False, search_after=('foo', 'doc#bar')) ) with mock.patch.object(ManifestService, 'get_manifest') as mock_get_manifest: for i, expected_status in enumerate(3 * [301] + [302]): response = requests.get(str(url), allow_redirects=False) if fetch: self.assertEqual(200, response.status_code) response = response.json() self.assertEqual(expected_status, response.pop('Status')) headers = response else: self.assertEqual(expected_status, response.status_code) headers = response.headers if expected_status == 301: self.assertGreaterEqual(int(headers['Retry-After']), 0) url = furl(headers['Location']) if i == 0: state = dict(format_=format_.value, catalog=self.catalog, filters=filters.to_json(), object_key=self.object_key, authentication=None, partition=partitions[0].to_json()) mock_helper.start_execution.assert_called_once_with( state_machine_name, execution_id, execution_input=state ) mock_helper.describe_execution.assert_not_called() mock_helper.reset_mock() mock_helper.describe_execution.return_value = {'status': 'RUNNING'} elif i == 1: mock_get_manifest.return_value = partitions[1] state = self.app_module.generate_manifest(state, None) self.assertEqual(partitions[1], ManifestPartition.from_json(state['partition'])) mock_get_manifest.assert_called_once_with( format_=ManifestFormat(state['format_']), catalog=state['catalog'], filters=Filters.from_json(state['filters']), partition=partitions[0], authentication=None, object_key=state['object_key'] ) mock_get_manifest.reset_mock() mock_helper.start_execution.assert_not_called() mock_helper.describe_execution.assert_called_once() mock_helper.reset_mock() # simulate absence of output due eventual consistency mock_helper.describe_execution.return_value = {'status': 'SUCCEEDED'} elif i == 2: mock_get_manifest.return_value = manifest mock_helper.start_execution.assert_not_called() mock_helper.describe_execution.assert_called_once() mock_helper.reset_mock() mock_helper.describe_execution.return_value = { 'status': 'SUCCEEDED', 'output': json.dumps( self.app_module.generate_manifest(state, None) ) } elif i == 3: mock_get_manifest.assert_called_once_with( format_=ManifestFormat(state['format_']), catalog=state['catalog'], filters=Filters.from_json(state['filters']), partition=partitions[1], authentication=None, object_key=state['object_key'] ) mock_get_manifest.reset_mock() mock_helper.start_execution.assert_not_called() mock_helper.describe_execution.assert_called_once() expected_url = str(manifest_url) if fetch else object_url self.assertEqual(expected_url, str(url)) mock_helper.reset_mock() manifest_states = [ manifest, CachedManifestNotFound, CachedManifestSourcesChanged ] with mock.patch.object(ManifestService, 'get_cached_manifest_with_object_key', side_effect=manifest_states): for manifest in manifest_states: with self.subTest(manifest=manifest): self.assertEqual(object_key, manifest_url.args['objectKey']) response = requests.get(str(manifest_url), allow_redirects=False) if isinstance(manifest, Manifest): self.assertEqual(302, response.status_code) self.assertEqual(object_url, response.headers['Location']) else: if manifest is CachedManifestNotFound: cause = 'expired' elif manifest is CachedManifestSourcesChanged: cause = 'become invalid due to an authorization change' else: assert False msg = f'GoneError: The requested manifest has {cause}, please request a new one' self.assertEqual(410, response.status_code) self.assertEqual(msg, response.json()['Message'])
from azul import ( config, ) from azul.deployment import ( aws, ) from azul.modules import ( load_app_module, ) direct_access_role = config.dss_direct_access_role('service') service = load_app_module('service') policy = { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents" ], "Resource": "arn:aws:logs:*:*:*" }, { "Effect": "Allow", "Action": [ "es:ESHttpDelete", "es:ESHttpGet", "es:ESHttpHead", "es:ESHttpPut", "es:ESHttpPost", "es:ESHttpDelete" ],
def on_modified(self, event): if event.src_path == self.tracked_file: self.service = load_app_module('service') write_specs(self.service.app) print('Spec updated')