def _generate_transform_request():
transform_request = TransformRequest()
transform_request.submit_time = 1000
transform_request.request_id = 'BR549'
transform_request.columns = 'electron.eta(), muon.pt()'
transform_request.tree_name = 'Events'
transform_request.chunk_size = 1000
transform_request.workers = 42
transform_request.did = '123-456-789'
transform_request.image = 'ssl-hep/foo:latest'
transform_request.result_destination = 'kafka'
transform_request.result_format = 'arrow'
transform_request.kafka_broker = 'http://ssl-hep.org.kafka:12345'
transform_request.total_events = 10000
transform_request.total_bytes = 1203
return transform_request
def post(self):
try:
transformation_request = parser.parse_args()
print("object store ", self.object_store)
request_id = str(uuid.uuid4())
time = datetime.now(tz=timezone.utc)
requested_did = transformation_request['did'] \
if 'did' in transformation_request else None
requested_file_list = transformation_request['file-list'] \
if 'file-list' in transformation_request else None
# requested_did xor requested_file_list
if bool(requested_did) == bool(requested_file_list):
raise BadRequest("Must provide did or file-list but not both")
if self.object_store and \
transformation_request['result-destination'] == 'object-store':
self.object_store.create_bucket(request_id)
# WHat happens if object-store and object_store is None?
if transformation_request['result-destination'] == 'kafka':
broker = transformation_request['kafka']['broker']
else:
broker = None
request_rec = TransformRequest(
did=requested_did
if requested_did else "File List Provided in Request",
submit_time=time,
columns=transformation_request['columns'],
selection=transformation_request['selection'],
tree_name=transformation_request['tree-name'],
request_id=str(request_id),
image=transformation_request['image'],
chunk_size=transformation_request['chunk-size'],
result_destination=transformation_request[
'result-destination'],
result_format=transformation_request['result-format'],
kafka_broker=broker,
workers=transformation_request['workers'],
workflow_name=_workflow_name(transformation_request))
# If we are doing the xaod_cpp workflow, then the first thing to do is make
# sure the requested selection is correct, and generate the C++ files
if request_rec.workflow_name == 'selection_codegen':
namespace = current_app.config['TRANSFORMER_NAMESPACE']
request_rec.generated_code_cm = \
self.code_gen_service.generate_code_for_selection(request_rec, namespace)
# Create queue for transformers to read from
self.rabbitmq_adaptor.setup_queue(request_id)
self.rabbitmq_adaptor.bind_queue_to_exchange(
exchange="transformation_requests", queue=request_id)
# Also setup an error queue for dead letters generated by transformer
self.rabbitmq_adaptor.setup_queue(request_id + "_errors")
self.rabbitmq_adaptor.bind_queue_to_exchange(
exchange="transformation_failures",
queue=request_id + "_errors")
request_rec.save_to_db()
if requested_did:
did_request = {
"request_id":
request_rec.request_id,
"did":
request_rec.did,
"service-endpoint":
self._generate_advertised_endpoint(
"servicex/transformation/" + request_rec.request_id)
}
self.rabbitmq_adaptor.basic_publish(
exchange='',
routing_key='did_requests',
body=json.dumps(did_request))
else:
# Request a preflight check on the first file
self.lookup_result_processor.publish_preflight_request(
request_rec, requested_file_list[0])
for file_path in requested_file_list:
file_record = DatasetFile(request_id=request_id,
file_path=file_path,
adler32="xxx",
file_events=0,
file_size=0)
self.lookup_result_processor.add_file_to_dataset(
request_rec, file_record)
self.lookup_result_processor.report_fileset_complete(
request_rec, num_files=len(requested_file_list))
db.session.commit()
if self.elasticsearch_adapter:
self.elasticsearch_adapter.create_update_request(
request_id,
self._generate_transformation_record(
request_rec, "locating DID"))
return {"request_id": str(request_id)}
except BadRequest as bad_request:
return {
'message':
f'The json request was malformed: {str(bad_request)}'
}, 400
except ValueError as eek:
return {
'message': f'Failed to submit transform request: {str(eek)}'
}, 400
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback, limit=20, file=sys.stdout)
print(exc_value)
return {'message': 'Something went wrong'}, 500
def _generate_test_request(self):
transform_request = TransformRequest()
transform_request.request_id = "462-33"
transform_request.selection = "test-string"
return transform_request