def test_add_file_to_dataset_kafka(self, mocker, mock_rabbit_adaptor):
processor = LookupResultProcessor(mock_rabbit_adaptor, None,
"http://cern.analysis.ch:5000/")
dataset_file = DatasetFile(request_id="BR549",
file_path="/foo/bar.root",
adler32='12345',
file_size=1024,
file_events=500)
dataset_file.id = 42
dataset_file.save_to_db = mocker.Mock()
processor.add_file_to_dataset(self._generate_transform_request(),
dataset_file)
dataset_file.save_to_db.assert_called()
mock_rabbit_adaptor.basic_publish.assert_called_with(
exchange='transformation_requests',
routing_key='BR549',
body=json.dumps({
"request-id": 'BR549',
"file-id": 42,
"columns": 'electron.eta(), muon.pt()',
"file-path": "/foo/bar.root",
"tree-name": "Events",
"service-endpoint":
"http://cern.analysis.ch:5000/servicex/transformation/BR549",
'result-destination': 'kafka',
'kafka-broker': 'http://ssl-hep.org.kafka:12345'
}))
def test_publish_preflight_request(self, mocker, mock_rabbit_adaptor):
processor = LookupResultProcessor(mock_rabbit_adaptor, None,
"http://foo.com/")
processor.publish_preflight_request(self._generate_transform_request(),
"root1.root")
mock_rabbit_adaptor.basic_publish.assert_called_with(
exchange='',
routing_key='validation_requests',
body=json.dumps({
"request-id":
'BR549',
"columns":
'electron.eta(), muon.pt()',
"file-path":
"root1.root",
"service-endpoint":
"http://foo.com/servicex/transformation/BR549"
}))
def test_report_fileset_complete(self, mocker, mock_rabbit_adaptor):
processor = LookupResultProcessor(mock_rabbit_adaptor, None,
"http://cern.analysis.ch:5000/")
transform_request = self._generate_transform_request()
processor.report_fileset_complete(transform_request,
num_files=1,
num_skipped=2,
total_events=3,
total_bytes=4,
did_lookup_time=5)
assert transform_request.files == 1
assert transform_request.files_skipped == 2
assert transform_request.total_events == 3
assert transform_request.total_bytes == 4
assert transform_request.did_lookup_time == 5
def test_report_fileset_complete(self, mocker, mock_rabbit_adaptor):
import servicex
mock_update = mocker.patch.object(servicex.models.TransformRequest,
"update_request")
processor = LookupResultProcessor(mock_rabbit_adaptor, None,
"http://cern.analysis.ch:5000/")
processor.report_fileset_complete(self._generate_transform_request(),
num_files=1,
num_skipped=2,
total_events=3,
total_bytes=4,
did_lookup_time=5)
mock_update.assert_called()
args = mock_update.call_args
assert args[0][0].files == 1
assert args[0][0].files_skipped == 2
assert args[0][0].total_events == 3
assert args[0][0].total_bytes == 4
assert args[0][0].did_lookup_time == 5
def create_app(test_config=None,
provided_transformer_manager=None,
provided_rabbit_adaptor=None,
provided_object_store=None,
provided_elasticsearch_adapter=None,
provided_code_gen_service=None,
provided_lookup_result_processor=None):
"""Create and configure an instance of the Flask application."""
app = Flask(__name__, instance_relative_config=True)
if not test_config:
app.config.from_envvar('APP_CONFIG_FILE')
else:
app.config.from_mapping(test_config)
print("Transformer enabled: ",
test_config['TRANSFORMER_MANAGER_ENABLED'])
with app.app_context():
if app.config['OBJECT_STORE_ENABLED']:
if not provided_object_store:
object_store = ObjectStoreManager(
app.config['MINIO_URL'],
username=app.config['MINIO_ACCESS_KEY'],
password=app.config['MINIO_SECRET_KEY'])
else:
object_store = provided_object_store
else:
object_store = None
if app.config[
'TRANSFORMER_MANAGER_ENABLED'] and not provided_transformer_manager:
transformer_manager = TransformerManager(
app.config['TRANSFORMER_MANAGER_MODE'])
else:
transformer_manager = provided_transformer_manager
if not provided_rabbit_adaptor:
rabbit_adaptor = _init_rabbit_mq(
app.config['RABBIT_MQ_URL'], app.config['RABBIT_RETRIES'],
app.config['RABBIT_RETRY_INTERVAL'])
else:
rabbit_adaptor = provided_rabbit_adaptor
if not provided_code_gen_service:
code_gen_service = CodeGenAdapter(
app.config['CODE_GEN_SERVICE_URL'], transformer_manager)
else:
code_gen_service = provided_code_gen_service
if 'ELASTIC_SEARCH_LOGGING_ENABLED' in app.config \
and app.config['ELASTIC_SEARCH_LOGGING_ENABLED']\
and not provided_elasticsearch_adapter:
elasticsearch_adaptor = ElasticSearchAdapter(
app.config['ES_HOST'], app.config['ES_PORT'],
app.config['ES_USER'], app.config['ES_PASS'])
else:
elasticsearch_adaptor = provided_elasticsearch_adapter
if not provided_lookup_result_processor:
lookup_result_processor = LookupResultProcessor(
rabbit_adaptor, elasticsearch_adaptor,
"http://" + app.config['ADVERTISED_HOSTNAME'] + "/")
else:
lookup_result_processor = provided_lookup_result_processor
api = Api(app)
# ensure the instance folder exists
try:
os.makedirs(app.instance_path)
except OSError:
pass
@app.before_first_request
def create_tables():
from servicex.models import db
db.init_app(app)
db.create_all()
add_routes(api, transformer_manager, rabbit_adaptor, object_store,
elasticsearch_adaptor, code_gen_service,
lookup_result_processor)
return app
def create_app(test_config=None,
provided_transformer_manager=None,
provided_rabbit_adaptor=None,
provided_object_store=None,
provided_elasticsearch_adapter=None,
provided_code_gen_service=None,
provided_lookup_result_processor=None,
provided_docker_repo_adapter=None):
"""Create and configure an instance of the Flask application."""
app = Flask(__name__, instance_relative_config=True)
JWTManager(app)
if not test_config:
app.config.from_envvar('APP_CONFIG_FILE')
else:
app.config.from_mapping(test_config)
print("Transformer enabled: ",
test_config['TRANSFORMER_MANAGER_ENABLED'])
with app.app_context():
if app.config['OBJECT_STORE_ENABLED']:
if not provided_object_store:
object_store = ObjectStoreManager(
app.config['MINIO_URL'],
username=app.config['MINIO_ACCESS_KEY'],
password=app.config['MINIO_SECRET_KEY'])
else:
object_store = provided_object_store
else:
object_store = None
if app.config[
'TRANSFORMER_MANAGER_ENABLED'] and not provided_transformer_manager:
transformer_manager = TransformerManager(
app.config['TRANSFORMER_MANAGER_MODE'])
else:
transformer_manager = provided_transformer_manager
if not provided_rabbit_adaptor:
rabbit_adaptor = _init_rabbit_mq(
app.config['RABBIT_MQ_URL'], app.config['RABBIT_RETRIES'],
app.config['RABBIT_RETRY_INTERVAL'])
else:
rabbit_adaptor = provided_rabbit_adaptor
if not provided_code_gen_service:
code_gen_service = CodeGenAdapter(
app.config['CODE_GEN_SERVICE_URL'], transformer_manager)
else:
code_gen_service = provided_code_gen_service
if 'ELASTIC_SEARCH_LOGGING_ENABLED' in app.config \
and app.config['ELASTIC_SEARCH_LOGGING_ENABLED']\
and not provided_elasticsearch_adapter:
elasticsearch_adaptor = ElasticSearchAdapter(
app.config['ES_HOST'], app.config['ES_PORT'],
app.config['ES_USER'], app.config['ES_PASS'])
else:
elasticsearch_adaptor = provided_elasticsearch_adapter
if not provided_lookup_result_processor:
lookup_result_processor = LookupResultProcessor(
rabbit_adaptor, elasticsearch_adaptor,
"http://" + app.config['ADVERTISED_HOSTNAME'] + "/")
else:
lookup_result_processor = provided_lookup_result_processor
if not provided_docker_repo_adapter:
docker_repo_adapter = DockerRepoAdapter()
else:
docker_repo_adapter = provided_docker_repo_adapter
api = Api(app)
# ensure the instance folder exists
try:
os.makedirs(app.instance_path)
except OSError:
pass
@app.before_first_request
def create_tables():
from servicex.models import db, UserModel
from flask_jwt_extended import (create_refresh_token,
create_access_token)
db.init_app(app)
db.create_all()
if not UserModel.find_by_email(app.config['JWT_ADMIN']):
try:
new_user = UserModel(email=app.config['JWT_ADMIN'],
full_name="Administrator",
key=UserModel.generate_hash(
app.config['JWT_PASS']),
admin=True,
pending=False)
new_user.save_to_db()
create_access_token(identity=app.config['JWT_ADMIN'])
create_refresh_token(identity=app.config['JWT_PASS'])
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback,
limit=20,
file=sys.stdout)
print(exc_value)
add_routes(api, transformer_manager, rabbit_adaptor, object_store,
elasticsearch_adaptor, code_gen_service,
lookup_result_processor, docker_repo_adapter)
app.add_url_rule("/", "index", index)
return app