def test_add_file_to_dataset_kafka(self, mocker, mock_rabbit_adaptor): processor = LookupResultProcessor(mock_rabbit_adaptor, None, "http://cern.analysis.ch:5000/") dataset_file = DatasetFile(request_id="BR549", file_path="/foo/bar.root", adler32='12345', file_size=1024, file_events=500) dataset_file.id = 42 dataset_file.save_to_db = mocker.Mock() processor.add_file_to_dataset(self._generate_transform_request(), dataset_file) dataset_file.save_to_db.assert_called() mock_rabbit_adaptor.basic_publish.assert_called_with( exchange='transformation_requests', routing_key='BR549', body=json.dumps({ "request-id": 'BR549', "file-id": 42, "columns": 'electron.eta(), muon.pt()', "file-path": "/foo/bar.root", "tree-name": "Events", "service-endpoint": "http://cern.analysis.ch:5000/servicex/transformation/BR549", 'result-destination': 'kafka', 'kafka-broker': 'http://ssl-hep.org.kafka:12345' }))
def test_publish_preflight_request(self, mocker, mock_rabbit_adaptor): processor = LookupResultProcessor(mock_rabbit_adaptor, None, "http://foo.com/") processor.publish_preflight_request(self._generate_transform_request(), "root1.root") mock_rabbit_adaptor.basic_publish.assert_called_with( exchange='', routing_key='validation_requests', body=json.dumps({ "request-id": 'BR549', "columns": 'electron.eta(), muon.pt()', "file-path": "root1.root", "service-endpoint": "http://foo.com/servicex/transformation/BR549" }))
def test_report_fileset_complete(self, mocker, mock_rabbit_adaptor): processor = LookupResultProcessor(mock_rabbit_adaptor, None, "http://cern.analysis.ch:5000/") transform_request = self._generate_transform_request() processor.report_fileset_complete(transform_request, num_files=1, num_skipped=2, total_events=3, total_bytes=4, did_lookup_time=5) assert transform_request.files == 1 assert transform_request.files_skipped == 2 assert transform_request.total_events == 3 assert transform_request.total_bytes == 4 assert transform_request.did_lookup_time == 5
def test_report_fileset_complete(self, mocker, mock_rabbit_adaptor): import servicex mock_update = mocker.patch.object(servicex.models.TransformRequest, "update_request") processor = LookupResultProcessor(mock_rabbit_adaptor, None, "http://cern.analysis.ch:5000/") processor.report_fileset_complete(self._generate_transform_request(), num_files=1, num_skipped=2, total_events=3, total_bytes=4, did_lookup_time=5) mock_update.assert_called() args = mock_update.call_args assert args[0][0].files == 1 assert args[0][0].files_skipped == 2 assert args[0][0].total_events == 3 assert args[0][0].total_bytes == 4 assert args[0][0].did_lookup_time == 5
def create_app(test_config=None, provided_transformer_manager=None, provided_rabbit_adaptor=None, provided_object_store=None, provided_elasticsearch_adapter=None, provided_code_gen_service=None, provided_lookup_result_processor=None): """Create and configure an instance of the Flask application.""" app = Flask(__name__, instance_relative_config=True) if not test_config: app.config.from_envvar('APP_CONFIG_FILE') else: app.config.from_mapping(test_config) print("Transformer enabled: ", test_config['TRANSFORMER_MANAGER_ENABLED']) with app.app_context(): if app.config['OBJECT_STORE_ENABLED']: if not provided_object_store: object_store = ObjectStoreManager( app.config['MINIO_URL'], username=app.config['MINIO_ACCESS_KEY'], password=app.config['MINIO_SECRET_KEY']) else: object_store = provided_object_store else: object_store = None if app.config[ 'TRANSFORMER_MANAGER_ENABLED'] and not provided_transformer_manager: transformer_manager = TransformerManager( app.config['TRANSFORMER_MANAGER_MODE']) else: transformer_manager = provided_transformer_manager if not provided_rabbit_adaptor: rabbit_adaptor = _init_rabbit_mq( app.config['RABBIT_MQ_URL'], app.config['RABBIT_RETRIES'], app.config['RABBIT_RETRY_INTERVAL']) else: rabbit_adaptor = provided_rabbit_adaptor if not provided_code_gen_service: code_gen_service = CodeGenAdapter( app.config['CODE_GEN_SERVICE_URL'], transformer_manager) else: code_gen_service = provided_code_gen_service if 'ELASTIC_SEARCH_LOGGING_ENABLED' in app.config \ and app.config['ELASTIC_SEARCH_LOGGING_ENABLED']\ and not provided_elasticsearch_adapter: elasticsearch_adaptor = ElasticSearchAdapter( app.config['ES_HOST'], app.config['ES_PORT'], app.config['ES_USER'], app.config['ES_PASS']) else: elasticsearch_adaptor = provided_elasticsearch_adapter if not provided_lookup_result_processor: lookup_result_processor = LookupResultProcessor( rabbit_adaptor, elasticsearch_adaptor, "http://" + app.config['ADVERTISED_HOSTNAME'] + "/") else: lookup_result_processor = provided_lookup_result_processor api = Api(app) # ensure the instance folder exists try: os.makedirs(app.instance_path) except OSError: pass @app.before_first_request def create_tables(): from servicex.models import db db.init_app(app) db.create_all() add_routes(api, transformer_manager, rabbit_adaptor, object_store, elasticsearch_adaptor, code_gen_service, lookup_result_processor) return app
def create_app(test_config=None, provided_transformer_manager=None, provided_rabbit_adaptor=None, provided_object_store=None, provided_elasticsearch_adapter=None, provided_code_gen_service=None, provided_lookup_result_processor=None, provided_docker_repo_adapter=None): """Create and configure an instance of the Flask application.""" app = Flask(__name__, instance_relative_config=True) JWTManager(app) if not test_config: app.config.from_envvar('APP_CONFIG_FILE') else: app.config.from_mapping(test_config) print("Transformer enabled: ", test_config['TRANSFORMER_MANAGER_ENABLED']) with app.app_context(): if app.config['OBJECT_STORE_ENABLED']: if not provided_object_store: object_store = ObjectStoreManager( app.config['MINIO_URL'], username=app.config['MINIO_ACCESS_KEY'], password=app.config['MINIO_SECRET_KEY']) else: object_store = provided_object_store else: object_store = None if app.config[ 'TRANSFORMER_MANAGER_ENABLED'] and not provided_transformer_manager: transformer_manager = TransformerManager( app.config['TRANSFORMER_MANAGER_MODE']) else: transformer_manager = provided_transformer_manager if not provided_rabbit_adaptor: rabbit_adaptor = _init_rabbit_mq( app.config['RABBIT_MQ_URL'], app.config['RABBIT_RETRIES'], app.config['RABBIT_RETRY_INTERVAL']) else: rabbit_adaptor = provided_rabbit_adaptor if not provided_code_gen_service: code_gen_service = CodeGenAdapter( app.config['CODE_GEN_SERVICE_URL'], transformer_manager) else: code_gen_service = provided_code_gen_service if 'ELASTIC_SEARCH_LOGGING_ENABLED' in app.config \ and app.config['ELASTIC_SEARCH_LOGGING_ENABLED']\ and not provided_elasticsearch_adapter: elasticsearch_adaptor = ElasticSearchAdapter( app.config['ES_HOST'], app.config['ES_PORT'], app.config['ES_USER'], app.config['ES_PASS']) else: elasticsearch_adaptor = provided_elasticsearch_adapter if not provided_lookup_result_processor: lookup_result_processor = LookupResultProcessor( rabbit_adaptor, elasticsearch_adaptor, "http://" + app.config['ADVERTISED_HOSTNAME'] + "/") else: lookup_result_processor = provided_lookup_result_processor if not provided_docker_repo_adapter: docker_repo_adapter = DockerRepoAdapter() else: docker_repo_adapter = provided_docker_repo_adapter api = Api(app) # ensure the instance folder exists try: os.makedirs(app.instance_path) except OSError: pass @app.before_first_request def create_tables(): from servicex.models import db, UserModel from flask_jwt_extended import (create_refresh_token, create_access_token) db.init_app(app) db.create_all() if not UserModel.find_by_email(app.config['JWT_ADMIN']): try: new_user = UserModel(email=app.config['JWT_ADMIN'], full_name="Administrator", key=UserModel.generate_hash( app.config['JWT_PASS']), admin=True, pending=False) new_user.save_to_db() create_access_token(identity=app.config['JWT_ADMIN']) create_refresh_token(identity=app.config['JWT_PASS']) except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_tb(exc_traceback, limit=20, file=sys.stdout) print(exc_value) add_routes(api, transformer_manager, rabbit_adaptor, object_store, elasticsearch_adaptor, code_gen_service, lookup_result_processor, docker_repo_adapter) app.add_url_rule("/", "index", index) return app